1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Thanks to
10 * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
11 * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
12 * Denes
13 * Chris Clark
14 * clach05
15 * Denes Lengyel
16 * and many others who have contributed to current and previous versions
17
18 This file contains the DAL support for many relational databases,
19 including:
20 - SQLite & SpatiaLite
21 - MySQL
22 - Postgres
23 - Firebird
24 - Oracle
25 - MS SQL
26 - DB2
27 - Interbase
28 - Ingres
29 - Informix (9+ and SE)
30 - SapDB (experimental)
31 - Cubrid (experimental)
32 - CouchDB (experimental)
33 - MongoDB (in progress)
34 - Google:nosql
35 - Google:sql
36 - Teradata
37 - IMAP (experimental)
38
39 Example of usage:
40
41 >>> # from dal import DAL, Field
42
43 ### create DAL connection (and create DB if it doesn't exist)
44 >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
45 ... folder=None)
46
47 ### define a table 'person' (create/alter as necessary)
48 >>> person = db.define_table('person',Field('name','string'))
49
50 ### insert a record
51 >>> id = person.insert(name='James')
52
53 ### retrieve it by id
54 >>> james = person(id)
55
56 ### retrieve it by name
57 >>> james = person(name='James')
58
59 ### retrieve it by arbitrary query
60 >>> query = (person.name=='James') & (person.name.startswith('J'))
61 >>> james = db(query).select(person.ALL)[0]
62
63 ### update one record
64 >>> james.update_record(name='Jim')
65 <Row {'id': 1, 'name': 'Jim'}>
66
67 ### update multiple records by query
68 >>> db(person.name.like('J%')).update(name='James')
69 1
70
71 ### delete records by query
72 >>> db(person.name.lower() == 'jim').delete()
73 0
74
75 ### retrieve multiple records (rows)
76 >>> people = db(person).select(orderby=person.name,
77 ... groupby=person.name, limitby=(0,100))
78
79 ### further filter them
80 >>> james = people.find(lambda row: row.name == 'James').first()
81 >>> print james.id, james.name
82 1 James
83
84 ### check aggregates
85 >>> counter = person.id.count()
86 >>> print db(person).select(counter).first()(counter)
87 1
88
89 ### delete one record
90 >>> james.delete_record()
91 1
92
93 ### delete (drop) entire database table
94 >>> person.drop()
95
96 Supported field types:
97 id string text boolean integer double decimal password upload
98 blob time date datetime
99
100 Supported DAL URI strings:
101 'sqlite://test.db'
102 'spatialite://test.db'
103 'sqlite:memory'
104 'spatialite:memory'
105 'jdbc:sqlite://test.db'
106 'mysql://root:none@localhost/test'
107 'postgres://mdipierro:password@localhost/test'
108 'postgres:psycopg2://mdipierro:password@localhost/test'
109 'postgres:pg8000://mdipierro:password@localhost/test'
110 'jdbc:postgres://mdipierro:none@localhost/test'
111 'mssql://web2py:none@A64X2/web2py_test'
112 'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
113 'oracle://username:password@database'
114 'firebird://user:password@server:3050/database'
115 'db2://DSN=dsn;UID=user;PWD=pass'
116 'firebird://username:password@hostname/database'
117 'firebird_embedded://username:password@c://path'
118 'informix://user:password@server:3050/database'
119 'informixu://user:password@server:3050/database' # unicode informix
120 'ingres://database' # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name'
121 'google:datastore' # for google app engine datastore
122 'google:sql' # for google app engine with sql (mysql compatible)
123 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
124 'imap://user:password@server:port' # experimental
125 'mongodb://user:password@server:port/database' # experimental
126
127 For more info:
128 help(DAL)
129 help(Field)
130 """
131
132
133
134
135
136 __all__ = ['DAL', 'Field']
137
138 DEFAULTLENGTH = {'string':512,
139 'password':512,
140 'upload':512,
141 'text':2**15,
142 'blob':2**31}
143 TIMINGSSIZE = 100
144 SPATIALLIBS = {
145 'Windows':'libspatialite',
146 'Linux':'libspatialite.so',
147 'Darwin':'libspatialite.dylib'
148 }
149 DEFAULT_URI = 'sqlite://dummy.db'
150
151 import re
152 import sys
153 import locale
154 import os
155 import types
156 import datetime
157 import threading
158 import time
159 import csv
160 import cgi
161 import copy
162 import socket
163 import logging
164 import base64
165 import shutil
166 import marshal
167 import decimal
168 import struct
169 import urllib
170 import hashlib
171 import uuid
172 import glob
173 import traceback
174 import platform
175
176 PYTHON_VERSION = sys.version_info[:3]
177 if PYTHON_VERSION[0] == 2:
178 import cPickle as pickle
179 import cStringIO as StringIO
180 import copy_reg as copyreg
181 hashlib_md5 = hashlib.md5
182 bytes, unicode = str, unicode
183 else:
184 import pickle
185 from io import StringIO as StringIO
186 import copyreg
187 long = int
188 hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
189 bytes, unicode = bytes, str
190
191 if PYTHON_VERSION[:2] < (2, 7):
192 from gluon.contrib.ordereddict import OrderedDict
193 else:
194 from collections import OrderedDict
195
196
197 CALLABLETYPES = (types.LambdaType, types.FunctionType,
198 types.BuiltinFunctionType,
199 types.MethodType, types.BuiltinMethodType)
200
201 TABLE_ARGS = set(
202 ('migrate','primarykey','fake_migrate','format','redefine',
203 'singular','plural','trigger_name','sequence_name','fields',
204 'common_filter','polymodel','table_class','on_define','rname'))
205
206 SELECT_ARGS = set(
207 ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
208 'distinct', 'having', 'join','for_update', 'processor','cacheable', 'orderby_on_limitby'))
209
210 ogetattr = object.__getattribute__
211 osetattr = object.__setattr__
212 exists = os.path.exists
213 pjoin = os.path.join
214
215
216
217
218 try:
219 from gluon.utils import web2py_uuid
220 except (ImportError, SystemError):
221 import uuid
223
224 try:
225 import portalocker
226 have_portalocker = True
227 except ImportError:
228 have_portalocker = False
229
230 try:
231 from gluon import serializers
232 have_serializers = True
233 except ImportError:
234 have_serializers = False
235 try:
236 import json as simplejson
237 except ImportError:
238 try:
239 import gluon.contrib.simplejson as simplejson
240 except ImportError:
241 simplejson = None
242
243 LOGGER = logging.getLogger("web2py.dal")
244 DEFAULT = lambda:0
245
246 GLOBAL_LOCKER = threading.RLock()
247 THREAD_LOCAL = threading.local()
248
249
250
251
252 REGEX_TYPE = re.compile('^([\w\_\:]+)')
253 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
254 REGEX_W = re.compile('^\w+$')
255 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$')
256 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)(\.(?P<name>\w+))?\.\w+$')
257 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
258 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
259 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
260 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
261 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
262 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
263 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
264 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
265 REGEX_QUOTES = re.compile("'[^']*'")
266 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
267 REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
268 REGEX_NOPASSWD = re.compile('\/\/[\w\.\-]+[\:\/](.+)(?=@)')
269
270
271
272 DRIVERS = []
273
274 try:
275 from new import classobj
276 from google.appengine.ext import db as gae
277 from google.appengine.ext import ndb
278 from google.appengine.api import namespace_manager, rdbms
279 from google.appengine.api.datastore_types import Key
280 from google.appengine.ext.db.polymodel import PolyModel
281 from google.appengine.ext.ndb.polymodel import PolyModel as NDBPolyModel
282 DRIVERS.append('google')
283 except ImportError:
284 pass
285
286 if not 'google' in DRIVERS:
287
288 try:
289 from pysqlite2 import dbapi2 as sqlite2
290 DRIVERS.append('SQLite(sqlite2)')
291 except ImportError:
292 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
293
294 try:
295 from sqlite3 import dbapi2 as sqlite3
296 DRIVERS.append('SQLite(sqlite3)')
297 except ImportError:
298 LOGGER.debug('no SQLite drivers sqlite3')
299
300 try:
301
302 try:
303 import gluon.contrib.pymysql as pymysql
304
305
306 pymysql.ESCAPE_REGEX = re.compile("'")
307 pymysql.ESCAPE_MAP = {"'": "''"}
308
309 except ImportError:
310 import pymysql
311 DRIVERS.append('MySQL(pymysql)')
312 except ImportError:
313 LOGGER.debug('no MySQL driver pymysql')
314
315 try:
316 import MySQLdb
317 DRIVERS.append('MySQL(MySQLdb)')
318 except ImportError:
319 LOGGER.debug('no MySQL driver MySQLDB')
320
321 try:
322 import mysql.connector as mysqlconnector
323 DRIVERS.append("MySQL(mysqlconnector)")
324 except ImportError:
325 LOGGER.debug("no driver mysql.connector")
326
327 try:
328 import psycopg2
329 from psycopg2.extensions import adapt as psycopg2_adapt
330 DRIVERS.append('PostgreSQL(psycopg2)')
331 except ImportError:
332 LOGGER.debug('no PostgreSQL driver psycopg2')
333
334 try:
335
336 try:
337 import gluon.contrib.pg8000.dbapi as pg8000
338 except ImportError:
339 import pg8000.dbapi as pg8000
340 DRIVERS.append('PostgreSQL(pg8000)')
341 except ImportError:
342 LOGGER.debug('no PostgreSQL driver pg8000')
343
344 try:
345 import cx_Oracle
346 DRIVERS.append('Oracle(cx_Oracle)')
347 except ImportError:
348 LOGGER.debug('no Oracle driver cx_Oracle')
349
350 try:
351 try:
352 import pyodbc
353 except ImportError:
354 try:
355 import gluon.contrib.pypyodbc as pyodbc
356 except Exception, e:
357 raise ImportError(str(e))
358 DRIVERS.append('MSSQL(pyodbc)')
359 DRIVERS.append('DB2(pyodbc)')
360 DRIVERS.append('Teradata(pyodbc)')
361 DRIVERS.append('Ingres(pyodbc)')
362 except ImportError:
363 LOGGER.debug('no MSSQL/DB2/Teradata/Ingres driver pyodbc')
364
365 try:
366 import Sybase
367 DRIVERS.append('Sybase(Sybase)')
368 except ImportError:
369 LOGGER.debug('no Sybase driver')
370
371 try:
372 import kinterbasdb
373 DRIVERS.append('Interbase(kinterbasdb)')
374 DRIVERS.append('Firebird(kinterbasdb)')
375 except ImportError:
376 LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
377
378 try:
379 import fdb
380 DRIVERS.append('Firebird(fdb)')
381 except ImportError:
382 LOGGER.debug('no Firebird driver fdb')
383
384 try:
385 import firebirdsql
386 DRIVERS.append('Firebird(firebirdsql)')
387 except ImportError:
388 LOGGER.debug('no Firebird driver firebirdsql')
389
390 try:
391 import informixdb
392 DRIVERS.append('Informix(informixdb)')
393 LOGGER.warning('Informix support is experimental')
394 except ImportError:
395 LOGGER.debug('no Informix driver informixdb')
396
397 try:
398 import sapdb
399 DRIVERS.append('SQL(sapdb)')
400 LOGGER.warning('SAPDB support is experimental')
401 except ImportError:
402 LOGGER.debug('no SAP driver sapdb')
403
404 try:
405 import cubriddb
406 DRIVERS.append('Cubrid(cubriddb)')
407 LOGGER.warning('Cubrid support is experimental')
408 except ImportError:
409 LOGGER.debug('no Cubrid driver cubriddb')
410
411 try:
412 from com.ziclix.python.sql import zxJDBC
413 import java.sql
414
415 from org.sqlite import JDBC
416 zxJDBC_sqlite = java.sql.DriverManager
417 DRIVERS.append('PostgreSQL(zxJDBC)')
418 DRIVERS.append('SQLite(zxJDBC)')
419 LOGGER.warning('zxJDBC support is experimental')
420 is_jdbc = True
421 except ImportError:
422 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
423 is_jdbc = False
424
425 try:
426 import couchdb
427 DRIVERS.append('CouchDB(couchdb)')
428 except ImportError:
429 LOGGER.debug('no Couchdb driver couchdb')
430
431 try:
432 import pymongo
433 DRIVERS.append('MongoDB(pymongo)')
434 except:
435 LOGGER.debug('no MongoDB driver pymongo')
436
437 try:
438 import imaplib
439 DRIVERS.append('IMAP(imaplib)')
440 except:
441 LOGGER.debug('no IMAP driver imaplib')
442
443 PLURALIZE_RULES = [
444 (re.compile('child$'), re.compile('child$'), 'children'),
445 (re.compile('oot$'), re.compile('oot$'), 'eet'),
446 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
447 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
448 (re.compile('sis$'), re.compile('sis$'), 'ses'),
449 (re.compile('man$'), re.compile('man$'), 'men'),
450 (re.compile('ife$'), re.compile('ife$'), 'ives'),
451 (re.compile('eau$'), re.compile('eau$'), 'eaux'),
452 (re.compile('lf$'), re.compile('lf$'), 'lves'),
453 (re.compile('[sxz]$'), re.compile('$'), 'es'),
454 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
455 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
456 (re.compile('$'), re.compile('$'), 's'),
457 ]
464
469
472
475
477
480
482 regex = re.compile('\.keyword(?=\w)')
483 a = regex.sub('."%s"' % keyword,a)
484 return a
485
486 if 'google' in DRIVERS:
487
488 is_jdbc = False
491 """
492 GAE decimal implementation
493 """
494 data_type = decimal.Decimal
495
496 - def __init__(self, precision, scale, **kwargs):
497 super(GAEDecimalProperty, self).__init__(self, **kwargs)
498 d = '1.'
499 for x in range(scale):
500 d += '0'
501 self.round = decimal.Decimal(d)
502
510
512 if value is None or value == '':
513 return None
514 else:
515 return decimal.Decimal(value).quantize(self.round)
516
518 value = super(GAEDecimalProperty, self).validate(value)
519 if value is None or isinstance(value, decimal.Decimal):
520 return value
521 elif isinstance(value, basestring):
522 return decimal.Decimal(value)
523 raise gae.BadValueError("Property %s must be a Decimal or string."\
524 % self.name)
525
528 """
529 NDB decimal implementation
530 """
531 data_type = decimal.Decimal
532
533 - def __init__(self, precision, scale, **kwargs):
534 d = '1.'
535 for x in range(scale):
536 d += '0'
537 self.round = decimal.Decimal(d)
538
540 if value is None or value == '':
541 return None
542 else:
543 return str(value)
544
546 if value is None or value == '':
547 return None
548 else:
549 return decimal.Decimal(value).quantize(self.round)
550
552 if value is None or isinstance(value, decimal.Decimal):
553 return value
554 elif isinstance(value, basestring):
555 return decimal.Decimal(value)
556 raise TypeError("Property %s must be a Decimal or string."\
557 % self._name)
558
564
565 POOLS = {}
566 check_active_connection = True
567
568 @staticmethod
571
572
573
574 - def close(self,action='commit',really=True):
591
592 @staticmethod
594 """ to close cleanly databases in a multithreaded environment """
595 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
596 for db_uid, db_group in dbs:
597 for db in db_group:
598 if hasattr(db,'_adapter'):
599 db._adapter.close(action)
600 getattr(THREAD_LOCAL,'db_instances',{}).clear()
601 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
602 if callable(action):
603 action(None)
604 return
605
618
620 """hook for the after_connection parameter"""
621 if callable(self._after_connection):
622 self._after_connection(self)
623 self.after_connection()
624
626 """ this it is supposed to be overloaded by adapters"""
627 pass
628
630 """
631 this function defines: self.connection and self.cursor
632 (iff cursor is True)
633 if self.pool_size>0 it will try pull the connection from the pool
634 if the connection is not active (closed by db server) it will loop
635 if not self.pool_size or no active connections in pool makes a new one
636 """
637 if getattr(self,'connection', None) != None:
638 return
639 if f is None:
640 f = self.connector
641
642
643
644
645
646 if not self.pool_size:
647 self.connection = f()
648 self.cursor = cursor and self.connection.cursor()
649 else:
650 uri = self.uri
651 POOLS = ConnectionPool.POOLS
652 while True:
653 GLOBAL_LOCKER.acquire()
654 if not uri in POOLS:
655 POOLS[uri] = []
656 if POOLS[uri]:
657 self.connection = POOLS[uri].pop()
658 GLOBAL_LOCKER.release()
659 self.cursor = cursor and self.connection.cursor()
660 try:
661 if self.cursor and self.check_active_connection:
662 self.execute('SELECT 1;')
663 break
664 except:
665 pass
666 else:
667 GLOBAL_LOCKER.release()
668 self.connection = f()
669 self.cursor = cursor and self.connection.cursor()
670 break
671 self.after_connection_hook()
672
679 native_json = False
680 driver = None
681 driver_name = None
682 drivers = ()
683 connection = None
684 commit_on_alter_table = False
685 support_distributed_transaction = False
686 uploads_in_blob = False
687 can_select_for_update = True
688 dbpath = None
689 folder = None
690
691 TRUE = 'T'
692 FALSE = 'F'
693 T_SEP = ' '
694 QUOTE_TEMPLATE = '"%s"'
695
696 types = {
697 'boolean': 'CHAR(1)',
698 'string': 'CHAR(%(length)s)',
699 'text': 'TEXT',
700 'json': 'TEXT',
701 'password': 'CHAR(%(length)s)',
702 'blob': 'BLOB',
703 'upload': 'CHAR(%(length)s)',
704 'integer': 'INTEGER',
705 'bigint': 'INTEGER',
706 'float':'DOUBLE',
707 'double': 'DOUBLE',
708 'decimal': 'DOUBLE',
709 'date': 'DATE',
710 'time': 'TIME',
711 'datetime': 'TIMESTAMP',
712 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
713 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
714 'list:integer': 'TEXT',
715 'list:string': 'TEXT',
716 'list:reference': 'TEXT',
717
718 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT',
719 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
720 }
721
723 if not hasattr(self.driver, "OperationalError"):
724 return None
725 return isinstance(exception, self.driver.OperationalError)
726
728 if not hasattr(self.driver, "ProgrammingError"):
729 return None
730 return isinstance(exception, self.driver.ProgrammingError)
731
733 pkeys = getattr(table,'_primarykey',None)
734 if pkeys:
735 return table[pkeys[0]] != None
736 else:
737 return table._id != None
738
740 return "'%s'" % obj.replace("'", "''")
741
743 if isinstance(obj,(int,float)):
744 return str(obj)
745 return self.adapt(str(obj))
746
748 """
749 to be used ONLY for files that on GAE may not be on filesystem
750 """
751 return exists(filename)
752
753 - def file_open(self, filename, mode='rb', lock=True):
754 """
755 to be used ONLY for files that on GAE may not be on filesystem
756 """
757 if have_portalocker and lock:
758 fileobj = portalocker.LockedFile(filename,mode)
759 else:
760 fileobj = open(filename,mode)
761 return fileobj
762
764 """
765 to be used ONLY for files that on GAE may not be on filesystem
766 """
767 if fileobj:
768 fileobj.close()
769
772
774 self.adapter_args = adapter_args
775 if getattr(self,'driver',None) != None:
776 return
777 drivers_available = [driver for driver in self.drivers
778 if driver in globals()]
779 if uri:
780 items = uri.split('://',1)[0].split(':')
781 request_driver = items[1] if len(items)>1 else None
782 else:
783 request_driver = None
784 request_driver = request_driver or adapter_args.get('driver')
785 if request_driver:
786 if request_driver in drivers_available:
787 self.driver_name = request_driver
788 self.driver = globals().get(request_driver)
789 else:
790 raise RuntimeError("driver %s not available" % request_driver)
791 elif drivers_available:
792 self.driver_name = drivers_available[0]
793 self.driver = globals().get(self.driver_name)
794 else:
795 raise RuntimeError("no driver available %s" % str(self.drivers))
796
797 - def log(self, message, table=None):
798 """ Logs migrations
799
800 It will not log changes if logfile is not specified. Defaults
801 to sql.log
802 """
803
804 isabs = None
805 logfilename = self.adapter_args.get('logfile','sql.log')
806 writelog = bool(logfilename)
807 if writelog:
808 isabs = os.path.isabs(logfilename)
809
810 if table and table._dbt and writelog and self.folder:
811 if isabs:
812 table._loggername = logfilename
813 else:
814 table._loggername = pjoin(self.folder, logfilename)
815 logfile = self.file_open(table._loggername, 'a')
816 logfile.write(message)
817 self.file_close(logfile)
818
819
820 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
821 credential_decoder=IDENTITY, driver_args={},
822 adapter_args={},do_connect=True, after_connection=None):
823 self.db = db
824 self.dbengine = "None"
825 self.uri = uri
826 self.pool_size = pool_size
827 self.folder = folder
828 self.db_codec = db_codec
829 self._after_connection = after_connection
830 class Dummy(object):
831 lastrowid = 1
832 def __getattr__(self, value):
833 return lambda *a, **b: []
834 self.connection = Dummy()
835 self.cursor = Dummy()
836
838 return '%s_sequence' % tablename
839
841 return '%s_sequence' % tablename
842
845
846 - def create_table(self, table,
847 migrate=True,
848 fake_migrate=False,
849 polymodel=None):
850 db = table._db
851 fields = []
852
853 postcreation_fields = []
854 sql_fields = {}
855 sql_fields_aux = {}
856 TFK = {}
857 tablename = table._tablename
858 sortable = 0
859 types = self.types
860 for field in table:
861 sortable += 1
862 field_name = field.name
863 field_type = field.type
864 if isinstance(field_type,SQLCustomType):
865 ftype = field_type.native or field_type.type
866 elif field_type.startswith('reference'):
867 referenced = field_type[10:].strip()
868 if referenced == '.':
869 referenced = tablename
870 constraint_name = self.constraint_name(tablename, field_name)
871 if not '.' in referenced \
872 and referenced != tablename \
873 and hasattr(table,'_primarykey'):
874 ftype = types['integer']
875 else:
876 if hasattr(table,'_primarykey'):
877 rtablename,rfieldname = referenced.split('.')
878 rtable = db[rtablename]
879 rfield = rtable[rfieldname]
880
881 if rfieldname in rtable._primarykey or \
882 rfield.unique:
883 ftype = types[rfield.type[:9]] % \
884 dict(length=rfield.length)
885
886 if not rfield.unique and len(rtable._primarykey)>1:
887
888 if rtablename not in TFK:
889 TFK[rtablename] = {}
890 TFK[rtablename][rfieldname] = field_name
891 else:
892 ftype = ftype + \
893 types['reference FK'] % dict(
894 constraint_name = constraint_name,
895 foreign_key = '%s (%s)' % (rtablename,
896 rfieldname),
897 table_name = tablename,
898 field_name = field._rname or field.name,
899 on_delete_action=field.ondelete)
900 else:
901
902 if referenced in db:
903 id_fieldname = db[referenced]._id.name
904 elif referenced == tablename:
905 id_fieldname = table._id.name
906 else:
907 id_fieldname = 'id'
908
909
910
911
912
913
914 real_referenced = (
915 (db[referenced]._rname or db[referenced])
916 if referenced == tablename or referenced in db
917 else referenced)
918
919 ftype = types[field_type[:9]] % dict(
920 index_name = field_name+'__idx',
921 field_name = field._rname or field.name,
922 constraint_name = constraint_name,
923 foreign_key = '%s (%s)' % (real_referenced,
924 id_fieldname),
925 on_delete_action=field.ondelete)
926 elif field_type.startswith('list:reference'):
927 ftype = types[field_type[:14]]
928 elif field_type.startswith('decimal'):
929 precision, scale = map(int,field_type[8:-1].split(','))
930 ftype = types[field_type[:7]] % \
931 dict(precision=precision,scale=scale)
932 elif field_type.startswith('geo'):
933 if not hasattr(self,'srid'):
934 raise RuntimeError('Adapter does not support geometry')
935 srid = self.srid
936 geotype, parms = field_type[:-1].split('(')
937 if not geotype in types:
938 raise SyntaxError(
939 'Field: unknown field type: %s for %s' \
940 % (field_type, field_name))
941 ftype = types[geotype]
942 if self.dbengine == 'postgres' and geotype == 'geometry':
943
944 dimension = 2
945 parms = parms.split(',')
946 if len(parms) == 3:
947 schema, srid, dimension = parms
948 elif len(parms) == 2:
949 schema, srid = parms
950 else:
951 schema = parms[0]
952 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
953 ftype = ftype % dict(schema=schema,
954 tablename=tablename,
955 fieldname=field_name, srid=srid,
956 dimension=dimension)
957 postcreation_fields.append(ftype)
958 elif not field_type in types:
959 raise SyntaxError('Field: unknown field type: %s for %s' % \
960 (field_type, field_name))
961 else:
962 ftype = types[field_type]\
963 % dict(length=field.length)
964 if not field_type.startswith('id') and \
965 not field_type.startswith('reference'):
966 if field.notnull:
967 ftype += ' NOT NULL'
968 else:
969 ftype += self.ALLOW_NULL()
970 if field.unique:
971 ftype += ' UNIQUE'
972 if field.custom_qualifier:
973 ftype += ' %s' % field.custom_qualifier
974
975
976 sql_fields[field_name] = dict(
977 length=field.length,
978 unique=field.unique,
979 notnull=field.notnull,
980 sortable=sortable,
981 type=str(field_type),
982 sql=ftype)
983
984 if field.notnull and not field.default is None:
985
986
987
988
989
990
991 not_null = self.NOT_NULL(field.default, field_type)
992 ftype = ftype.replace('NOT NULL', not_null)
993 sql_fields_aux[field_name] = dict(sql=ftype)
994
995
996 if not (self.dbengine == 'postgres' and \
997 field_type.startswith('geom')):
998
999 field_rname = "%s" % (field._rname or field_name)
1000 fields.append('%s %s' % (field_rname, ftype))
1001 other = ';'
1002
1003
1004 if self.dbengine == 'mysql':
1005 if not hasattr(table, "_primarykey"):
1006 fields.append('PRIMARY KEY(%s)' % (table._id.name or table._id._rname))
1007 other = ' ENGINE=InnoDB CHARACTER SET utf8;'
1008
1009 fields = ',\n '.join(fields)
1010 for rtablename in TFK:
1011 rfields = TFK[rtablename]
1012 pkeys = db[rtablename]._primarykey
1013 fkeys = [ rfields[k] for k in pkeys ]
1014 fields = fields + ',\n ' + \
1015 types['reference TFK'] % dict(
1016 table_name = tablename,
1017 field_name=', '.join(fkeys),
1018 foreign_table = rtablename,
1019 foreign_key = ', '.join(pkeys),
1020 on_delete_action = field.ondelete)
1021
1022 table_rname = table._rname or tablename
1023
1024 if getattr(table,'_primarykey',None):
1025 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \
1026 (table_rname, fields,
1027 self.PRIMARY_KEY(', '.join(table._primarykey)),other)
1028 else:
1029 query = "CREATE TABLE %s(\n %s\n)%s" % \
1030 (table_rname, fields, other)
1031
1032 if self.uri.startswith('sqlite:///') \
1033 or self.uri.startswith('spatialite:///'):
1034 path_encoding = sys.getfilesystemencoding() \
1035 or locale.getdefaultlocale()[1] or 'utf8'
1036 dbpath = self.uri[9:self.uri.rfind('/')]\
1037 .decode('utf8').encode(path_encoding)
1038 else:
1039 dbpath = self.folder
1040
1041 if not migrate:
1042 return query
1043 elif self.uri.startswith('sqlite:memory')\
1044 or self.uri.startswith('spatialite:memory'):
1045 table._dbt = None
1046 elif isinstance(migrate, str):
1047 table._dbt = pjoin(dbpath, migrate)
1048 else:
1049 table._dbt = pjoin(
1050 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
1051
1052 if not table._dbt or not self.file_exists(table._dbt):
1053 if table._dbt:
1054 self.log('timestamp: %s\n%s\n'
1055 % (datetime.datetime.today().isoformat(),
1056 query), table)
1057 if not fake_migrate:
1058 self.create_sequence_and_triggers(query,table)
1059 table._db.commit()
1060
1061
1062 for query in postcreation_fields:
1063 self.execute(query)
1064 table._db.commit()
1065 if table._dbt:
1066 tfile = self.file_open(table._dbt, 'w')
1067 pickle.dump(sql_fields, tfile)
1068 self.file_close(tfile)
1069 if fake_migrate:
1070 self.log('faked!\n', table)
1071 else:
1072 self.log('success!\n', table)
1073 else:
1074 tfile = self.file_open(table._dbt, 'r')
1075 try:
1076 sql_fields_old = pickle.load(tfile)
1077 except EOFError:
1078 self.file_close(tfile)
1079 raise RuntimeError('File %s appears corrupted' % table._dbt)
1080 self.file_close(tfile)
1081 if sql_fields != sql_fields_old:
1082 self.migrate_table(
1083 table,
1084 sql_fields, sql_fields_old,
1085 sql_fields_aux, None,
1086 fake_migrate=fake_migrate
1087 )
1088 return query
1089
1090 - def migrate_table(
1091 self,
1092 table,
1093 sql_fields,
1094 sql_fields_old,
1095 sql_fields_aux,
1096 logfile,
1097 fake_migrate=False,
1098 ):
1099
1100
1101 db = table._db
1102 db._migrated.append(table._tablename)
1103 tablename = table._tablename
1104 def fix(item):
1105 k,v=item
1106 if not isinstance(v,dict):
1107 v=dict(type='unknown',sql=v)
1108 return k.lower(),v
1109
1110
1111 sql_fields = dict(map(fix,sql_fields.iteritems()))
1112 sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
1113 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
1114 if db._debug:
1115 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields))
1116
1117 keys = sql_fields.keys()
1118 for key in sql_fields_old:
1119 if not key in keys:
1120 keys.append(key)
1121 new_add = self.concat_add(tablename)
1122
1123 metadata_change = False
1124 sql_fields_current = copy.copy(sql_fields_old)
1125 for key in keys:
1126 query = None
1127 if not key in sql_fields_old:
1128 sql_fields_current[key] = sql_fields[key]
1129 if self.dbengine in ('postgres',) and \
1130 sql_fields[key]['type'].startswith('geometry'):
1131
1132 query = [ sql_fields[key]['sql'] ]
1133 else:
1134 query = ['ALTER TABLE %s ADD %s %s;' % \
1135 (tablename, key,
1136 sql_fields_aux[key]['sql'].replace(', ', new_add))]
1137 metadata_change = True
1138 elif self.dbengine in ('sqlite', 'spatialite'):
1139 if key in sql_fields:
1140 sql_fields_current[key] = sql_fields[key]
1141 metadata_change = True
1142 elif not key in sql_fields:
1143 del sql_fields_current[key]
1144 ftype = sql_fields_old[key]['type']
1145 if (self.dbengine in ('postgres',) and
1146 ftype.startswith('geometry')):
1147 geotype, parms = ftype[:-1].split('(')
1148 schema = parms.split(',')[0]
1149 query = [ "SELECT DropGeometryColumn ('%(schema)s', "+
1150 "'%(table)s', '%(field)s');" %
1151 dict(schema=schema, table=tablename, field=key,) ]
1152 elif self.dbengine in ('firebird',):
1153 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
1154 else:
1155 query = ['ALTER TABLE %s DROP COLUMN %s;' %
1156 (tablename, key)]
1157 metadata_change = True
1158 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
1159 and not (key in table.fields and
1160 isinstance(table[key].type, SQLCustomType)) \
1161 and not sql_fields[key]['type'].startswith('reference')\
1162 and not sql_fields[key]['type'].startswith('double')\
1163 and not sql_fields[key]['type'].startswith('id'):
1164 sql_fields_current[key] = sql_fields[key]
1165 t = tablename
1166 tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
1167 if self.dbengine in ('firebird',):
1168 drop_expr = 'ALTER TABLE %s DROP %s;'
1169 else:
1170 drop_expr = 'ALTER TABLE %s DROP COLUMN %s;'
1171 key_tmp = key + '__tmp'
1172 query = ['ALTER TABLE %s ADD %s %s;' % (t, key_tmp, tt),
1173 'UPDATE %s SET %s=%s;' % (t, key_tmp, key),
1174 drop_expr % (t, key),
1175 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1176 'UPDATE %s SET %s=%s;' % (t, key, key_tmp),
1177 drop_expr % (t, key_tmp)]
1178 metadata_change = True
1179 elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
1180 sql_fields_current[key] = sql_fields[key]
1181 metadata_change = True
1182
1183 if query:
1184 self.log('timestamp: %s\n'
1185 % datetime.datetime.today().isoformat(), table)
1186 db['_lastsql'] = '\n'.join(query)
1187 for sub_query in query:
1188 self.log(sub_query + '\n', table)
1189 if fake_migrate:
1190 if db._adapter.commit_on_alter_table:
1191 self.save_dbt(table,sql_fields_current)
1192 self.log('faked!\n', table)
1193 else:
1194 self.execute(sub_query)
1195
1196
1197
1198
1199
1200 if db._adapter.commit_on_alter_table:
1201 db.commit()
1202 self.save_dbt(table,sql_fields_current)
1203 self.log('success!\n', table)
1204
1205 elif metadata_change:
1206 self.save_dbt(table,sql_fields_current)
1207
1208 if metadata_change and not (query and db._adapter.commit_on_alter_table):
1209 db.commit()
1210 self.save_dbt(table,sql_fields_current)
1211 self.log('success!\n', table)
1212
1213 - def save_dbt(self,table, sql_fields_current):
1214 tfile = self.file_open(table._dbt, 'w')
1215 pickle.dump(sql_fields_current, tfile)
1216 self.file_close(tfile)
1217
1218 - def LOWER(self, first):
1220
1221 - def UPPER(self, first):
1223
1224 - def COUNT(self, first, distinct=None):
1225 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \
1226 % self.expand(first)
1227
1229 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1230
1231 - def EPOCH(self, first):
1233
1236
1239
1242
1245
1248
1249 - def NOT_NULL(self, default, field_type):
1250 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1251
1253 expressions = [self.expand(first)]+[self.expand(e) for e in second]
1254 return 'COALESCE(%s)' % ','.join(expressions)
1255
1258
1259 - def RAW(self, first):
1261
1264
1266 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1267
1269 return 'PRIMARY KEY(%s)' % key
1270
1271 - def _drop(self, table, mode):
1272 table_rname = table._rname or table
1273 return ['DROP TABLE %s;' % table_rname]
1274
1275 - def drop(self, table, mode=''):
1289
1290 - def _insert(self, table, fields):
1298
1300 table_rname = table._rname or table
1301 return 'INSERT INTO %s DEFAULT VALUES;' % table_rname
1302
1303 - def insert(self, table, fields):
1304 query = self._insert(table,fields)
1305 try:
1306 self.execute(query)
1307 except Exception:
1308 e = sys.exc_info()[1]
1309 if hasattr(table,'_on_insert_error'):
1310 return table._on_insert_error(table,fields,e)
1311 raise e
1312 if hasattr(table,'_primarykey'):
1313 return dict([(k[0].name, k[1]) for k in fields \
1314 if k[0].name in table._primarykey])
1315 id = self.lastrowid(table)
1316 if not isinstance(id, (int, long)):
1317 return id
1318 rid = Reference(id)
1319 (rid._table, rid._record) = (table, None)
1320 return rid
1321
1323 return [self.insert(table,item) for item in items]
1324
1325 - def NOT(self, first):
1327
1328 - def AND(self, first, second):
1330
1331 - def OR(self, first, second):
1333
1334 - def BELONGS(self, first, second):
1335 if isinstance(second, str):
1336 return '(%s IN (%s))' % (self.expand(first), second[:-1])
1337 if not second:
1338 return '(1=0)'
1339 items = ','.join(self.expand(item, first.type) for item in second)
1340 return '(%s IN (%s))' % (self.expand(first), items)
1341
1342 - def REGEXP(self, first, second):
1343 "regular expression operator"
1344 raise NotImplementedError
1345
1346 - def LIKE(self, first, second):
1347 "case sensitive like operator"
1348 raise NotImplementedError
1349
1350 - def ILIKE(self, first, second):
1351 "case in-sensitive like operator"
1352 return '(%s LIKE %s)' % (self.expand(first),
1353 self.expand(second, 'string'))
1354
1356 return '(%s LIKE %s)' % (self.expand(first),
1357 self.expand(second+'%', 'string'))
1358
1360 return '(%s LIKE %s)' % (self.expand(first),
1361 self.expand('%'+second, 'string'))
1362
1363 - def CONTAINS(self,first,second,case_sensitive=False):
1364 if first.type in ('string','text', 'json'):
1365 if isinstance(second,Expression):
1366 second = Expression(None,self.CONCAT('%',Expression(
1367 None,self.REPLACE(second,('%','%%'))),'%'))
1368 else:
1369 second = '%'+str(second).replace('%','%%')+'%'
1370 elif first.type.startswith('list:'):
1371 if isinstance(second,Expression):
1372 second = Expression(None,self.CONCAT(
1373 '%|',Expression(None,self.REPLACE(
1374 Expression(None,self.REPLACE(
1375 second,('%','%%'))),('|','||'))),'|%'))
1376 else:
1377 second = '%|'+str(second).replace('%','%%')\
1378 .replace('|','||')+'|%'
1379 op = case_sensitive and self.LIKE or self.ILIKE
1380 return op(first,second)
1381
1382 - def EQ(self, first, second=None):
1387
1388 - def NE(self, first, second=None):
1393
1394 - def LT(self,first,second=None):
1395 if second is None:
1396 raise RuntimeError("Cannot compare %s < None" % first)
1397 return '(%s < %s)' % (self.expand(first),
1398 self.expand(second,first.type))
1399
1400 - def LE(self,first,second=None):
1401 if second is None:
1402 raise RuntimeError("Cannot compare %s <= None" % first)
1403 return '(%s <= %s)' % (self.expand(first),
1404 self.expand(second,first.type))
1405
1406 - def GT(self,first,second=None):
1407 if second is None:
1408 raise RuntimeError("Cannot compare %s > None" % first)
1409 return '(%s > %s)' % (self.expand(first),
1410 self.expand(second,first.type))
1411
1412 - def GE(self,first,second=None):
1413 if second is None:
1414 raise RuntimeError("Cannot compare %s >= None" % first)
1415 return '(%s >= %s)' % (self.expand(first),
1416 self.expand(second,first.type))
1417
1419 return ftype in ('integer','boolean','double','bigint') or \
1420 ftype.startswith('decimal')
1421
1422 - def REPLACE(self, first, (second, third)):
1423 return 'REPLACE(%s,%s,%s)' % (self.expand(first,'string'),
1424 self.expand(second,'string'),
1425 self.expand(third,'string'))
1426
1429
1430 - def ADD(self, first, second):
1436
1437 - def SUB(self, first, second):
1440
1441 - def MUL(self, first, second):
1444
1445 - def DIV(self, first, second):
1448
1449 - def MOD(self, first, second):
1452
1453 - def AS(self, first, second):
1455
1456 - def ON(self, first, second):
1461
1464
1465 - def COMMA(self, first, second):
1467
1468 - def CAST(self, first, second):
1469 return 'CAST(%s AS %s)' % (first, second)
1470
1471 - def expand(self, expression, field_type=None, colnames=False):
1472 if isinstance(expression, Field):
1473 et = expression.table
1474 if not colnames:
1475 table_rname = et._ot and et._tablename or et._rname or et._tablename
1476 else:
1477 table_rname = et._tablename
1478 if not colnames:
1479 out = '%s.%s' % (table_rname, expression._rname or expression.name)
1480 else:
1481 out = '%s.%s' % (table_rname, expression.name)
1482 if field_type == 'string' and not expression.type in (
1483 'string','text','json','password'):
1484 out = self.CAST(out, self.types['text'])
1485 return out
1486 elif isinstance(expression, (Expression, Query)):
1487 first = expression.first
1488 second = expression.second
1489 op = expression.op
1490 optional_args = expression.optional_args or {}
1491 if not second is None:
1492 out = op(first, second, **optional_args)
1493 elif not first is None:
1494 out = op(first,**optional_args)
1495 elif isinstance(op, str):
1496 if op.endswith(';'):
1497 op=op[:-1]
1498 out = '(%s)' % op
1499 else:
1500 out = op()
1501 return out
1502 elif field_type:
1503 return str(self.represent(expression,field_type))
1504 elif isinstance(expression,(list,tuple)):
1505 return ','.join(self.represent(item,field_type) \
1506 for item in expression)
1507 elif isinstance(expression, bool):
1508 return '1' if expression else '0'
1509 else:
1510 return str(expression)
1511
1516
1517 - def alias(self, table, alias):
1518 """
1519 Given a table object, makes a new table object
1520 with alias name.
1521 """
1522 other = copy.copy(table)
1523 other['_ot'] = other._ot or other._rname or other._tablename
1524 other['ALL'] = SQLALL(other)
1525 other['_tablename'] = alias
1526 for fieldname in other.fields:
1527 other[fieldname] = copy.copy(other[fieldname])
1528 other[fieldname]._tablename = alias
1529 other[fieldname].tablename = alias
1530 other[fieldname].table = other
1531 table._db[alias] = other
1532 return other
1533
1535 tablename = table._rname or table._tablename
1536 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1537
1539
1540 try:
1541 queries = table._db._adapter._truncate(table, mode)
1542 for query in queries:
1543 self.log(query + '\n', table)
1544 self.execute(query)
1545 self.log('success!\n', table)
1546 finally:
1547 pass
1548
1549 - def _update(self, tablename, query, fields):
1550 if query:
1551 if use_common_filters(query):
1552 query = self.common_filter(query, [tablename])
1553 sql_w = ' WHERE ' + self.expand(query)
1554 else:
1555 sql_w = ''
1556 sql_v = ','.join(['%s=%s' % (field._rname or field.name,
1557 self.expand(value, field.type)) \
1558 for (field, value) in fields])
1559 tablename = "%s" % (self.db[tablename]._rname or tablename)
1560 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1561
1562 - def update(self, tablename, query, fields):
1563 sql = self._update(tablename, query, fields)
1564 try:
1565 self.execute(sql)
1566 except Exception:
1567 e = sys.exc_info()[1]
1568 table = self.db[tablename]
1569 if hasattr(table,'_on_update_error'):
1570 return table._on_update_error(table,query,fields,e)
1571 raise e
1572 try:
1573 return self.cursor.rowcount
1574 except:
1575 return None
1576
1577 - def _delete(self, tablename, query):
1578 if query:
1579 if use_common_filters(query):
1580 query = self.common_filter(query, [tablename])
1581 sql_w = ' WHERE ' + self.expand(query)
1582 else:
1583 sql_w = ''
1584 tablename = '%s' % (self.db[tablename]._rname or tablename)
1585 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1586
1587 - def delete(self, tablename, query):
1588 sql = self._delete(tablename, query)
1589
1590 db = self.db
1591 table = db[tablename]
1592 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by:
1593 deleted = [x[table._id.name] for x in db(query).select(table._id)]
1594
1595 self.execute(sql)
1596 try:
1597 counter = self.cursor.rowcount
1598 except:
1599 counter = None
1600
1601 if self.dbengine in ('sqlite', 'spatialite') and counter:
1602 for field in table._referenced_by:
1603 if field.type=='reference '+table._tablename \
1604 and field.ondelete=='CASCADE':
1605 db(field.belongs(deleted)).delete()
1606
1607 return counter
1608
1610 tablenames = self.tables(query)
1611 if len(tablenames)==1:
1612 return tablenames[0]
1613 elif len(tablenames)<1:
1614 raise RuntimeError("No table selected")
1615 else:
1616 raise RuntimeError("Too many tables selected")
1617
1619 db = self.db
1620 new_fields = []
1621 append = new_fields.append
1622 for item in fields:
1623 if isinstance(item,SQLALL):
1624 new_fields += item._table
1625 elif isinstance(item,str):
1626 if REGEX_TABLE_DOT_FIELD.match(item):
1627 tablename,fieldname = item.split('.')
1628 append(db[tablename][fieldname])
1629 else:
1630 append(Expression(db,lambda item=item:item))
1631 else:
1632 append(item)
1633
1634 if not new_fields:
1635 for table in tablenames:
1636 for field in db[table]:
1637 append(field)
1638 return new_fields
1639
1640 - def _select(self, query, fields, attributes):
1641 tables = self.tables
1642 for key in set(attributes.keys())-SELECT_ARGS:
1643 raise SyntaxError('invalid select attribute: %s' % key)
1644 args_get = attributes.get
1645 tablenames = tables(query)
1646 tablenames_for_common_filters = tablenames
1647 for field in fields:
1648 if isinstance(field, basestring) \
1649 and REGEX_TABLE_DOT_FIELD.match(field):
1650 tn,fn = field.split('.')
1651 field = self.db[tn][fn]
1652 for tablename in tables(field):
1653 if not tablename in tablenames:
1654 tablenames.append(tablename)
1655
1656 if len(tablenames) < 1:
1657 raise SyntaxError('Set: no tables selected')
1658 def colexpand(field):
1659 return self.expand(field, colnames=True)
1660 self._colnames = map(colexpand, fields)
1661 def geoexpand(field):
1662 if isinstance(field.type,str) and field.type.startswith('geometry') and isinstance(field, Field):
1663 field = field.st_astext()
1664 return self.expand(field)
1665 sql_f = ', '.join(map(geoexpand, fields))
1666 sql_o = ''
1667 sql_s = ''
1668 left = args_get('left', False)
1669 inner_join = args_get('join', False)
1670 distinct = args_get('distinct', False)
1671 groupby = args_get('groupby', False)
1672 orderby = args_get('orderby', False)
1673 having = args_get('having', False)
1674 limitby = args_get('limitby', False)
1675 orderby_on_limitby = args_get('orderby_on_limitby', True)
1676 for_update = args_get('for_update', False)
1677 if self.can_select_for_update is False and for_update is True:
1678 raise SyntaxError('invalid select attribute: for_update')
1679 if distinct is True:
1680 sql_s += 'DISTINCT'
1681 elif distinct:
1682 sql_s += 'DISTINCT ON (%s)' % distinct
1683 if inner_join:
1684 icommand = self.JOIN()
1685 if not isinstance(inner_join, (tuple, list)):
1686 inner_join = [inner_join]
1687 ijoint = [t._tablename for t in inner_join
1688 if not isinstance(t,Expression)]
1689 ijoinon = [t for t in inner_join if isinstance(t, Expression)]
1690 itables_to_merge={}
1691 [itables_to_merge.update(
1692 dict.fromkeys(tables(t))) for t in ijoinon]
1693 ijoinont = [t.first._tablename for t in ijoinon]
1694 [itables_to_merge.pop(t) for t in ijoinont
1695 if t in itables_to_merge]
1696 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys()
1697 iexcluded = [t for t in tablenames
1698 if not t in iimportant_tablenames]
1699 if left:
1700 join = attributes['left']
1701 command = self.LEFT_JOIN()
1702 if not isinstance(join, (tuple, list)):
1703 join = [join]
1704 joint = [t._tablename for t in join
1705 if not isinstance(t, Expression)]
1706 joinon = [t for t in join if isinstance(t, Expression)]
1707
1708 tables_to_merge={}
1709 [tables_to_merge.update(
1710 dict.fromkeys(tables(t))) for t in joinon]
1711 joinont = [t.first._tablename for t in joinon]
1712 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
1713 tablenames_for_common_filters = [t for t in tablenames
1714 if not t in joinont ]
1715 important_tablenames = joint + joinont + tables_to_merge.keys()
1716 excluded = [t for t in tablenames
1717 if not t in important_tablenames ]
1718 else:
1719 excluded = tablenames
1720
1721 if use_common_filters(query):
1722 query = self.common_filter(query,tablenames_for_common_filters)
1723 sql_w = ' WHERE ' + self.expand(query) if query else ''
1724
1725 if inner_join and not left:
1726 sql_t = ', '.join([self.table_alias(t) for t in iexcluded + \
1727 itables_to_merge.keys()])
1728 for t in ijoinon:
1729 sql_t += ' %s %s' % (icommand, t)
1730 elif not inner_join and left:
1731 sql_t = ', '.join([self.table_alias(t) for t in excluded + \
1732 tables_to_merge.keys()])
1733 if joint:
1734 sql_t += ' %s %s' % (command,
1735 ','.join([self.table_alias(t) for t in joint]))
1736 for t in joinon:
1737 sql_t += ' %s %s' % (command, t)
1738 elif inner_join and left:
1739 all_tables_in_query = set(important_tablenames + \
1740 iimportant_tablenames + \
1741 tablenames)
1742 tables_in_joinon = set(joinont + ijoinont)
1743 tables_not_in_joinon = \
1744 all_tables_in_query.difference(tables_in_joinon)
1745 sql_t = ','.join([self.table_alias(t) for t in tables_not_in_joinon])
1746 for t in ijoinon:
1747 sql_t += ' %s %s' % (icommand, t)
1748 if joint:
1749 sql_t += ' %s %s' % (command,
1750 ','.join([self.table_alias(t) for t in joint]))
1751 for t in joinon:
1752 sql_t += ' %s %s' % (command, t)
1753 else:
1754 sql_t = ', '.join(self.table_alias(t) for t in tablenames)
1755 if groupby:
1756 if isinstance(groupby, (list, tuple)):
1757 groupby = xorify(groupby)
1758 sql_o += ' GROUP BY %s' % self.expand(groupby)
1759 if having:
1760 sql_o += ' HAVING %s' % attributes['having']
1761 if orderby:
1762 if isinstance(orderby, (list, tuple)):
1763 orderby = xorify(orderby)
1764 if str(orderby) == '<random>':
1765 sql_o += ' ORDER BY %s' % self.RANDOM()
1766 else:
1767 sql_o += ' ORDER BY %s' % self.expand(orderby)
1768 if (limitby and not groupby and tablenames and orderby_on_limitby and not orderby):
1769 sql_o += ' ORDER BY %s' % ', '.join(
1770 ['%s.%s'%(t,x) for t in tablenames for x in (
1771 hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey
1772 or [self.db[t]._id._rname or self.db[t]._id.name]
1773 )
1774 ]
1775 )
1776
1777 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
1778 if for_update and self.can_select_for_update is True:
1779 sql = sql.rstrip(';') + ' FOR UPDATE;'
1780 return sql
1781
1782 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1783 if limitby:
1784 (lmin, lmax) = limitby
1785 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
1786 return 'SELECT %s %s FROM %s%s%s;' % \
1787 (sql_s, sql_f, sql_t, sql_w, sql_o)
1788
1790 return self.cursor.fetchall()
1791
1793 args_get = attributes.get
1794 cache = args_get('cache',None)
1795 if not cache:
1796 self.execute(sql)
1797 rows = self._fetchall()
1798 else:
1799 (cache_model, time_expire) = cache
1800 key = self.uri + '/' + sql + '/rows'
1801 if len(key)>200: key = hashlib_md5(key).hexdigest()
1802 def _select_aux2():
1803 self.execute(sql)
1804 return self._fetchall()
1805 rows = cache_model(key,_select_aux2,time_expire)
1806 if isinstance(rows,tuple):
1807 rows = list(rows)
1808 limitby = args_get('limitby', None) or (0,)
1809 rows = self.rowslice(rows,limitby[0],None)
1810 processor = args_get('processor',self.parse)
1811 cacheable = args_get('cacheable',False)
1812 return processor(rows,fields,self._colnames,cacheable=cacheable)
1813
1814 - def select(self, query, fields, attributes):
1815 """
1816 Always returns a Rows object, possibly empty.
1817 """
1818 sql = self._select(query, fields, attributes)
1819 cache = attributes.get('cache', None)
1820 if cache and attributes.get('cacheable',False):
1821 del attributes['cache']
1822 (cache_model, time_expire) = cache
1823 key = self.uri + '/' + sql
1824 if len(key)>200: key = hashlib_md5(key).hexdigest()
1825 args = (sql,fields,attributes)
1826 return cache_model(
1827 key,
1828 lambda self=self,args=args:self._select_aux(*args),
1829 time_expire)
1830 else:
1831 return self._select_aux(sql,fields,attributes)
1832
1833 - def _count(self, query, distinct=None):
1834 tablenames = self.tables(query)
1835 if query:
1836 if use_common_filters(query):
1837 query = self.common_filter(query, tablenames)
1838 sql_w = ' WHERE ' + self.expand(query)
1839 else:
1840 sql_w = ''
1841 sql_t = ','.join(self.table_alias(t) for t in tablenames)
1842 if distinct:
1843 if isinstance(distinct,(list, tuple)):
1844 distinct = xorify(distinct)
1845 sql_d = self.expand(distinct)
1846 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \
1847 (sql_d, sql_t, sql_w)
1848 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1849
1850 - def count(self, query, distinct=None):
1851 self.execute(self._count(query, distinct))
1852 return self.cursor.fetchone()[0]
1853
1865
1869
1873
1879
1882
1885
1888
1891
1894
1896 return '%s_%s__constraint' % (table,fieldname)
1897
1900
1902 if not self.connection: return None
1903 command = a[0]
1904 if hasattr(self,'filter_sql_command'):
1905 command = self.filter_sql_command(command)
1906 if self.db._debug:
1907 LOGGER.debug('SQL: %s' % command)
1908 self.db._lastsql = command
1909 t0 = time.time()
1910 ret = self.cursor.execute(command, *a[1:], **b)
1911 self.db._timings.append((command,time.time()-t0))
1912 del self.db._timings[:-TIMINGSSIZE]
1913 return ret
1914
1917
1919 field_is_type = fieldtype.startswith
1920 if isinstance(obj, CALLABLETYPES):
1921 obj = obj()
1922 if isinstance(fieldtype, SQLCustomType):
1923 value = fieldtype.encoder(obj)
1924 if fieldtype.type in ('string','text', 'json'):
1925 return self.adapt(value)
1926 return value
1927 if isinstance(obj, (Expression, Field)):
1928 return str(obj)
1929 if field_is_type('list:'):
1930 if not obj:
1931 obj = []
1932 elif not isinstance(obj, (list, tuple)):
1933 obj = [obj]
1934 if field_is_type('list:string'):
1935 obj = map(str,obj)
1936 else:
1937 obj = map(int,[o for o in obj if o != ''])
1938
1939 if isinstance(obj, (list, tuple)) and (not fieldtype == "json"):
1940 obj = bar_encode(obj)
1941 if obj is None:
1942 return 'NULL'
1943 if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']:
1944 return 'NULL'
1945 r = self.represent_exceptions(obj, fieldtype)
1946 if not r is None:
1947 return r
1948 if fieldtype == 'boolean':
1949 if obj and not str(obj)[:1].upper() in '0F':
1950 return self.smart_adapt(self.TRUE)
1951 else:
1952 return self.smart_adapt(self.FALSE)
1953 if fieldtype == 'id' or fieldtype == 'integer':
1954 return str(long(obj))
1955 if field_is_type('decimal'):
1956 return str(obj)
1957 elif field_is_type('reference'):
1958 if fieldtype.find('.')>0:
1959 return repr(obj)
1960 elif isinstance(obj, (Row, Reference)):
1961 return str(obj['id'])
1962 return str(long(obj))
1963 elif fieldtype == 'double':
1964 return repr(float(obj))
1965 if isinstance(obj, unicode):
1966 obj = obj.encode(self.db_codec)
1967 if fieldtype == 'blob':
1968 obj = base64.b64encode(str(obj))
1969 elif fieldtype == 'date':
1970 if isinstance(obj, (datetime.date, datetime.datetime)):
1971 obj = obj.isoformat()[:10]
1972 else:
1973 obj = str(obj)
1974 elif fieldtype == 'datetime':
1975 if isinstance(obj, datetime.datetime):
1976 obj = obj.isoformat(self.T_SEP)[:19]
1977 elif isinstance(obj, datetime.date):
1978 obj = obj.isoformat()[:10]+self.T_SEP+'00:00:00'
1979 else:
1980 obj = str(obj)
1981 elif fieldtype == 'time':
1982 if isinstance(obj, datetime.time):
1983 obj = obj.isoformat()[:10]
1984 else:
1985 obj = str(obj)
1986 elif fieldtype == 'json':
1987 if not self.native_json:
1988 if have_serializers:
1989 obj = serializers.json(obj)
1990 elif simplejson:
1991 obj = simplejson.dumps(obj)
1992 else:
1993 raise RuntimeError("missing simplejson")
1994 if not isinstance(obj,bytes):
1995 obj = bytes(obj)
1996 try:
1997 obj.decode(self.db_codec)
1998 except:
1999 obj = obj.decode('latin1').encode(self.db_codec)
2000 return self.adapt(obj)
2001
2004
2007
2008 - def rowslice(self, rows, minimum=0, maximum=None):
2009 """
2010 By default this function does nothing;
2011 overload when db does not do slicing.
2012 """
2013 return rows
2014
2015 - def parse_value(self, value, field_type, blob_decode=True):
2016 if field_type != 'blob' and isinstance(value, str):
2017 try:
2018 value = value.decode(self.db._db_codec)
2019 except Exception:
2020 pass
2021 if isinstance(value, unicode):
2022 value = value.encode('utf-8')
2023 if isinstance(field_type, SQLCustomType):
2024 value = field_type.decoder(value)
2025 if not isinstance(field_type, str) or value is None:
2026 return value
2027 elif field_type in ('string', 'text', 'password', 'upload', 'dict'):
2028 return value
2029 elif field_type.startswith('geo'):
2030 return value
2031 elif field_type == 'blob' and not blob_decode:
2032 return value
2033 else:
2034 key = REGEX_TYPE.match(field_type).group(0)
2035 return self.parsemap[key](value,field_type)
2036
2038 referee = field_type[10:].strip()
2039 if not '.' in referee:
2040 value = Reference(value)
2041 value._table, value._record = self.db[referee], None
2042 return value
2043
2045 return value == self.TRUE or str(value)[:1].lower() == 't'
2046
2048 if isinstance(value, datetime.datetime):
2049 return value.date()
2050 if not isinstance(value, (datetime.date,datetime.datetime)):
2051 (y, m, d) = map(int, str(value)[:10].strip().split('-'))
2052 value = datetime.date(y, m, d)
2053 return value
2054
2056 if not isinstance(value, datetime.time):
2057 time_items = map(int,str(value)[:8].strip().split(':')[:3])
2058 if len(time_items) == 3:
2059 (h, mi, s) = time_items
2060 else:
2061 (h, mi, s) = time_items + [0]
2062 value = datetime.time(h, mi, s)
2063 return value
2064
2066 if not isinstance(value, datetime.datetime):
2067 value = str(value)
2068 date_part,time_part,timezone = value[:10],value[11:19],value[19:]
2069 if '+' in timezone:
2070 ms,tz = timezone.split('+')
2071 h,m = tz.split(':')
2072 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m))
2073 elif '-' in timezone:
2074 ms,tz = timezone.split('-')
2075 h,m = tz.split(':')
2076 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m))
2077 else:
2078 dt = None
2079 (y, m, d) = map(int,date_part.split('-'))
2080 time_parts = time_part and time_part.split(':')[:3] or (0,0,0)
2081 while len(time_parts)<3: time_parts.append(0)
2082 time_items = map(int,time_parts)
2083 (h, mi, s) = time_items
2084 value = datetime.datetime(y, m, d, h, mi, s)
2085 if dt:
2086 value = value + dt
2087 return value
2088
2090 return base64.b64decode(str(value))
2091
2093 decimals = int(field_type[8:-1].split(',')[-1])
2094 if self.dbengine in ('sqlite', 'spatialite'):
2095 value = ('%.' + str(decimals) + 'f') % value
2096 if not isinstance(value, decimal.Decimal):
2097 value = decimal.Decimal(str(value))
2098 return value
2099
2104
2109
2114
2115 - def parse_id(self, value, field_type):
2117
2120
2123
2125 if not self.native_json:
2126 if not isinstance(value, basestring):
2127 raise RuntimeError('json data not a string')
2128 if isinstance(value, unicode):
2129 value = value.encode('utf-8')
2130 if have_serializers:
2131 value = serializers.loads_json(value)
2132 elif simplejson:
2133 value = simplejson.loads(value)
2134 else:
2135 raise RuntimeError("missing simplejson")
2136 return value
2137
2139 self.parsemap = {
2140 'id':self.parse_id,
2141 'integer':self.parse_integer,
2142 'bigint':self.parse_integer,
2143 'float':self.parse_double,
2144 'double':self.parse_double,
2145 'reference':self.parse_reference,
2146 'boolean':self.parse_boolean,
2147 'date':self.parse_date,
2148 'time':self.parse_time,
2149 'datetime':self.parse_datetime,
2150 'blob':self.parse_blob,
2151 'decimal':self.parse_decimal,
2152 'json':self.parse_json,
2153 'list:integer':self.parse_list_integers,
2154 'list:reference':self.parse_list_references,
2155 'list:string':self.parse_list_strings,
2156 }
2157
2158 - def parse(self, rows, fields, colnames, blob_decode=True,
2159 cacheable = False):
2160 db = self.db
2161 virtualtables = []
2162 new_rows = []
2163 tmps = []
2164 for colname in colnames:
2165 if not REGEX_TABLE_DOT_FIELD.match(colname):
2166 tmps.append(None)
2167 else:
2168 (tablename, _the_sep_, fieldname) = colname.partition('.')
2169 table = db[tablename]
2170 field = table[fieldname]
2171 ft = field.type
2172 tmps.append((tablename,fieldname,table,field,ft))
2173 for (i,row) in enumerate(rows):
2174 new_row = Row()
2175 for (j,colname) in enumerate(colnames):
2176 value = row[j]
2177 tmp = tmps[j]
2178 if tmp:
2179 (tablename,fieldname,table,field,ft) = tmp
2180 if tablename in new_row:
2181 colset = new_row[tablename]
2182 else:
2183 colset = new_row[tablename] = Row()
2184 if tablename not in virtualtables:
2185 virtualtables.append(tablename)
2186 value = self.parse_value(value,ft,blob_decode)
2187 if field.filter_out:
2188 value = field.filter_out(value)
2189 colset[fieldname] = value
2190
2191
2192 if ft=='id' and fieldname!='id' and \
2193 not 'id' in table.fields:
2194 colset['id'] = value
2195
2196 if ft == 'id' and not cacheable:
2197
2198
2199
2200 if isinstance(self, GoogleDatastoreAdapter):
2201 id = value.key.id() if self.use_ndb else value.key().id_or_name()
2202 colset[fieldname] = id
2203 colset.gae_item = value
2204 else:
2205 id = value
2206 colset.update_record = RecordUpdater(colset,table,id)
2207 colset.delete_record = RecordDeleter(table,id)
2208 if table._db._lazy_tables:
2209 colset['__get_lazy_reference__'] = LazyReferenceGetter(table, id)
2210 for rfield in table._referenced_by:
2211 referee_link = db._referee_name and \
2212 db._referee_name % dict(
2213 table=rfield.tablename,field=rfield.name)
2214 if referee_link and not referee_link in colset:
2215 colset[referee_link] = LazySet(rfield,id)
2216 else:
2217 if not '_extra' in new_row:
2218 new_row['_extra'] = Row()
2219 new_row['_extra'][colname] = \
2220 self.parse_value(value,
2221 fields[j].type,blob_decode)
2222 new_column_name = \
2223 REGEX_SELECT_AS_PARSER.search(colname)
2224 if not new_column_name is None:
2225 column_name = new_column_name.groups(0)
2226 setattr(new_row,column_name[0],value)
2227 new_rows.append(new_row)
2228 rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
2229
2230
2231 for tablename in virtualtables:
2232 table = db[tablename]
2233 fields_virtual = [(f,v) for (f,v) in table.iteritems()
2234 if isinstance(v,FieldVirtual)]
2235 fields_lazy = [(f,v) for (f,v) in table.iteritems()
2236 if isinstance(v,FieldMethod)]
2237 if fields_virtual or fields_lazy:
2238 for row in rowsobj.records:
2239 box = row[tablename]
2240 for f,v in fields_virtual:
2241 try:
2242 box[f] = v.f(row)
2243 except AttributeError:
2244 pass
2245 for f,v in fields_lazy:
2246 try:
2247 box[f] = (v.handler or VirtualCommand)(v.f,row)
2248 except AttributeError:
2249 pass
2250
2251
2252 for item in table.virtualfields:
2253 try:
2254 rowsobj = rowsobj.setvirtualfields(**{tablename:item})
2255 except (KeyError, AttributeError):
2256
2257 pass
2258 return rowsobj
2259
2261 tenant_fieldname = self.db._request_tenant
2262
2263 for tablename in tablenames:
2264 table = self.db[tablename]
2265
2266
2267 if table._common_filter != None:
2268 query = query & table._common_filter(query)
2269
2270
2271 if tenant_fieldname in table:
2272 default = table[tenant_fieldname].default
2273 if not default is None:
2274 newquery = table[tenant_fieldname] == default
2275 if query is None:
2276 query = newquery
2277 else:
2278 query = query & newquery
2279 return query
2280
2281 - def CASE(self,query,t,f):
2282 def represent(x):
2283 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'}
2284 if x is None: return 'NULL'
2285 elif isinstance(x,Expression): return str(x)
2286 else: return self.represent(x,types.get(type(x),'string'))
2287 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \
2288 (self.expand(query),represent(t),represent(f)))
2289
2295 drivers = ('sqlite2','sqlite3')
2296
2297 can_select_for_update = None
2298
2300 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2301
2302 @staticmethod
2304 table = {
2305 'year': (0, 4),
2306 'month': (5, 7),
2307 'day': (8, 10),
2308 'hour': (11, 13),
2309 'minute': (14, 16),
2310 'second': (17, 19),
2311 }
2312 try:
2313 if lookup != 'epoch':
2314 (i, j) = table[lookup]
2315 return int(s[i:j])
2316 else:
2317 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
2318 except:
2319 return None
2320
2321 @staticmethod
2323 return re.compile(expression).search(item) is not None
2324
2325 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2326 credential_decoder=IDENTITY, driver_args={},
2327 adapter_args={}, do_connect=True, after_connection=None):
2328 self.db = db
2329 self.dbengine = "sqlite"
2330 self.uri = uri
2331 self.adapter_args = adapter_args
2332 if do_connect: self.find_driver(adapter_args)
2333 self.pool_size = 0
2334 self.folder = folder
2335 self.db_codec = db_codec
2336 self._after_connection = after_connection
2337 self.find_or_make_work_folder()
2338 path_encoding = sys.getfilesystemencoding() \
2339 or locale.getdefaultlocale()[1] or 'utf8'
2340 if uri.startswith('sqlite:memory'):
2341 self.dbpath = ':memory:'
2342 else:
2343 self.dbpath = uri.split('://',1)[1]
2344 if self.dbpath[0] != '/':
2345 if PYTHON_VERSION[0] == 2:
2346 self.dbpath = pjoin(
2347 self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
2348 else:
2349 self.dbpath = pjoin(self.folder, self.dbpath)
2350 if not 'check_same_thread' in driver_args:
2351 driver_args['check_same_thread'] = False
2352 if not 'detect_types' in driver_args and do_connect:
2353 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2354 def connector(dbpath=self.dbpath, driver_args=driver_args):
2355 return self.driver.Connection(dbpath, **driver_args)
2356 self.connector = connector
2357 if do_connect: self.reconnect()
2358
2367
2369 tablename = table._tablename
2370 return ['DELETE FROM %s;' % tablename,
2371 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2372
2375
2376 - def REGEXP(self,first,second):
2377 return '(%s REGEXP %s)' % (self.expand(first),
2378 self.expand(second,'string'))
2379
2380 - def select(self, query, fields, attributes):
2381 """
2382 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
2383 Note that the entire database, rather than one record, is locked
2384 (it will be locked eventually anyway by the following UPDATE).
2385 """
2386 if attributes.get('for_update', False) and not 'cache' in attributes:
2387 self.execute('BEGIN IMMEDIATE TRANSACTION;')
2388 return super(SQLiteAdapter, self).select(query, fields, attributes)
2389
2391 drivers = ('sqlite3','sqlite2')
2392
2393 types = copy.copy(BaseAdapter.types)
2394 types.update(geometry='GEOMETRY')
2395
2396 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2397 credential_decoder=IDENTITY, driver_args={},
2398 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
2399 self.db = db
2400 self.dbengine = "spatialite"
2401 self.uri = uri
2402 if do_connect: self.find_driver(adapter_args)
2403 self.pool_size = 0
2404 self.folder = folder
2405 self.db_codec = db_codec
2406 self._after_connection = after_connection
2407 self.find_or_make_work_folder()
2408 self.srid = srid
2409 path_encoding = sys.getfilesystemencoding() \
2410 or locale.getdefaultlocale()[1] or 'utf8'
2411 if uri.startswith('spatialite:memory'):
2412 self.dbpath = ':memory:'
2413 else:
2414 self.dbpath = uri.split('://',1)[1]
2415 if self.dbpath[0] != '/':
2416 self.dbpath = pjoin(
2417 self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
2418 if not 'check_same_thread' in driver_args:
2419 driver_args['check_same_thread'] = False
2420 if not 'detect_types' in driver_args and do_connect:
2421 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2422 def connector(dbpath=self.dbpath, driver_args=driver_args):
2423 return self.driver.Connection(dbpath, **driver_args)
2424 self.connector = connector
2425 if do_connect: self.reconnect()
2426
2439
2440
2441
2443 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
2444 second['precision'], second['options'])
2445
2446 - def ST_ASTEXT(self, first):
2447 return 'AsText(%s)' %(self.expand(first))
2448
2452
2456
2460
2464
2468
2470 return 'Simplify(%s,%s)' %(self.expand(first),
2471 self.expand(second, 'double'))
2472
2476
2480
2482 field_is_type = fieldtype.startswith
2483 if field_is_type('geo'):
2484 srid = 4326
2485 geotype, parms = fieldtype[:-1].split('(')
2486 parms = parms.split(',')
2487 if len(parms) >= 2:
2488 schema, srid = parms[:2]
2489
2490 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2491
2492
2493
2494
2495 return value
2496 return BaseAdapter.represent(self, obj, fieldtype)
2497
2500 drivers = ('zxJDBC_sqlite',)
2501
2502 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
2503 credential_decoder=IDENTITY, driver_args={},
2504 adapter_args={}, do_connect=True, after_connection=None):
2527 self.connector = connector
2528 if do_connect: self.reconnect()
2529
2534
2537
2540 drivers = ('MySQLdb','pymysql', 'mysqlconnector')
2541
2542 commit_on_alter_table = True
2543 support_distributed_transaction = True
2544 types = {
2545 'boolean': 'CHAR(1)',
2546 'string': 'VARCHAR(%(length)s)',
2547 'text': 'LONGTEXT',
2548 'json': 'LONGTEXT',
2549 'password': 'VARCHAR(%(length)s)',
2550 'blob': 'LONGBLOB',
2551 'upload': 'VARCHAR(%(length)s)',
2552 'integer': 'INT',
2553 'bigint': 'BIGINT',
2554 'float': 'FLOAT',
2555 'double': 'DOUBLE',
2556 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2557 'date': 'DATE',
2558 'time': 'TIME',
2559 'datetime': 'DATETIME',
2560 'id': 'INT AUTO_INCREMENT NOT NULL',
2561 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2562 'list:integer': 'LONGTEXT',
2563 'list:string': 'LONGTEXT',
2564 'list:reference': 'LONGTEXT',
2565 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
2566 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2567 }
2568
2569 QUOTE_TEMPLATE = "`%s`"
2570
2573
2576
2578 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
2579 parameters[0], parameters[1])
2580
2581 - def EPOCH(self, first):
2583
2585 return 'CONCAT(%s)' % ','.join(self.expand(x,'string') for x in items)
2586
2587 - def REGEXP(self,first,second):
2588 return '(%s REGEXP %s)' % (self.expand(first),
2589 self.expand(second,'string'))
2590
2591 - def _drop(self,table,mode):
2592
2593 table_rname = table._rname or table
2594 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table_rname,
2595 'SET FOREIGN_KEY_CHECKS=1;']
2596
2598 return 'INSERT INTO %s VALUES (DEFAULT);' % table
2599
2602
2606
2609
2612
2613 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
2614
2615 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2616 credential_decoder=IDENTITY, driver_args={},
2617 adapter_args={}, do_connect=True, after_connection=None):
2618 self.db = db
2619 self.dbengine = "mysql"
2620 self.uri = uri
2621 if do_connect: self.find_driver(adapter_args,uri)
2622 self.pool_size = pool_size
2623 self.folder = folder
2624 self.db_codec = db_codec
2625 self._after_connection = after_connection
2626 self.find_or_make_work_folder()
2627 ruri = uri.split('://',1)[1]
2628 m = self.REGEX_URI.match(ruri)
2629 if not m:
2630 raise SyntaxError(
2631 "Invalid URI string in DAL: %s" % self.uri)
2632 user = credential_decoder(m.group('user'))
2633 if not user:
2634 raise SyntaxError('User required')
2635 password = credential_decoder(m.group('password'))
2636 if not password:
2637 password = ''
2638 host = m.group('host')
2639 if not host:
2640 raise SyntaxError('Host name required')
2641 db = m.group('db')
2642 if not db:
2643 raise SyntaxError('Database name required')
2644 port = int(m.group('port') or '3306')
2645 charset = m.group('charset') or 'utf8'
2646 driver_args.update(db=db,
2647 user=credential_decoder(user),
2648 passwd=credential_decoder(password),
2649 host=host,
2650 port=port,
2651 charset=charset)
2652
2653
2654 def connector(driver_args=driver_args):
2655 return self.driver.connect(**driver_args)
2656 self.connector = connector
2657 if do_connect: self.reconnect()
2658
2660 self.execute('SET FOREIGN_KEY_CHECKS=1;')
2661 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2662
2664 self.execute('select last_insert_id();')
2665 return int(self.cursor.fetchone()[0])
2666
2667
2668 -class PostgreSQLAdapter(BaseAdapter):
2669 drivers = ('psycopg2','pg8000')
2670
2671 support_distributed_transaction = True
2672 types = {
2673 'boolean': 'CHAR(1)',
2674 'string': 'VARCHAR(%(length)s)',
2675 'text': 'TEXT',
2676 'json': 'TEXT',
2677 'password': 'VARCHAR(%(length)s)',
2678 'blob': 'BYTEA',
2679 'upload': 'VARCHAR(%(length)s)',
2680 'integer': 'INTEGER',
2681 'bigint': 'BIGINT',
2682 'float': 'FLOAT',
2683 'double': 'FLOAT8',
2684 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2685 'date': 'DATE',
2686 'time': 'TIME',
2687 'datetime': 'TIMESTAMP',
2688 'id': 'SERIAL PRIMARY KEY',
2689 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2690 'list:integer': 'TEXT',
2691 'list:string': 'TEXT',
2692 'list:reference': 'TEXT',
2693 'geometry': 'GEOMETRY',
2694 'geography': 'GEOGRAPHY',
2695 'big-id': 'BIGSERIAL PRIMARY KEY',
2696 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2697 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2698 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2699
2700 }
2701
2702 QUOTE_TEMPLATE = '"%s"'
2703
2704 - def varquote(self,name):
2705 return varquote_aux(name,'"%s"')
2706
2707 - def adapt(self,obj):
2708 if self.driver_name == 'psycopg2':
2709 return psycopg2_adapt(obj).getquoted()
2710 elif self.driver_name == 'pg8000':
2711 return "'%s'" % str(obj).replace("%","%%").replace("'","''")
2712 else:
2713 return "'%s'" % str(obj).replace("'","''")
2714
2715 - def sequence_name(self,table):
2716 return '%s_id_seq' % table
2717
2720
2721 - def ADD(self, first, second):
2722 t = first.type
2723 if t in ('text','string','password', 'json', 'upload','blob'):
2724 return '(%s || %s)' % (self.expand(first), self.expand(second, t))
2725 else:
2726 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2727
2730
2731 - def prepare(self,key):
2732 self.execute("PREPARE TRANSACTION '%s';" % key)
2733
2734 - def commit_prepared(self,key):
2735 self.execute("COMMIT PREPARED '%s';" % key)
2736
2737 - def rollback_prepared(self,key):
2738 self.execute("ROLLBACK PREPARED '%s';" % key)
2739
2740 - def create_sequence_and_triggers(self, query, table, **args):
2741
2742
2743
2744
2745 self.execute(query)
2746
2747 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
2748
2749 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2750 credential_decoder=IDENTITY, driver_args={},
2751 adapter_args={}, do_connect=True, srid=4326,
2752 after_connection=None):
2753 self.db = db
2754 self.dbengine = "postgres"
2755 self.uri = uri
2756 if do_connect: self.find_driver(adapter_args,uri)
2757 self.pool_size = pool_size
2758 self.folder = folder
2759 self.db_codec = db_codec
2760 self._after_connection = after_connection
2761 self.srid = srid
2762 self.find_or_make_work_folder()
2763 ruri = uri.split('://',1)[1]
2764 m = self.REGEX_URI.match(ruri)
2765 if not m:
2766 raise SyntaxError("Invalid URI string in DAL")
2767 user = credential_decoder(m.group('user'))
2768 if not user:
2769 raise SyntaxError('User required')
2770 password = credential_decoder(m.group('password'))
2771 if not password:
2772 password = ''
2773 host = m.group('host')
2774 if not host:
2775 raise SyntaxError('Host name required')
2776 db = m.group('db')
2777 if not db:
2778 raise SyntaxError('Database name required')
2779 port = m.group('port') or '5432'
2780 sslmode = m.group('sslmode')
2781 if sslmode:
2782 msg = ("dbname='%s' user='%s' host='%s' "
2783 "port=%s password='%s' sslmode='%s'") \
2784 % (db, user, host, port, password, sslmode)
2785 else:
2786 msg = ("dbname='%s' user='%s' host='%s' "
2787 "port=%s password='%s'") \
2788 % (db, user, host, port, password)
2789
2790 if self.driver:
2791 self.__version__ = "%s %s" % (self.driver.__name__,
2792 self.driver.__version__)
2793 else:
2794 self.__version__ = None
2795 def connector(msg=msg,driver_args=driver_args):
2796 return self.driver.connect(msg,**driver_args)
2797 self.connector = connector
2798 if do_connect: self.reconnect()
2799
2800 - def after_connection(self):
2801 self.connection.set_client_encoding('UTF8')
2802 self.execute("SET standard_conforming_strings=on;")
2803 self.try_json()
2804
2805 - def lastrowid(self,table):
2806 self.execute("""select currval('"%s"')""" % table._sequence_name)
2807 return int(self.cursor.fetchone()[0])
2808
2809 - def try_json(self):
2810
2811
2812 if self.driver_name == "pg8000":
2813 supports_json = self.connection.server_version >= "9.2.0"
2814 elif (self.driver_name == "psycopg2") and \
2815 (self.driver.__version__ >= "2.0.12"):
2816 supports_json = self.connection.server_version >= 90200
2817 elif self.driver_name == "zxJDBC":
2818 supports_json = self.connection.dbversion >= "9.2.0"
2819 else: supports_json = None
2820 if supports_json:
2821 self.types["json"] = "JSON"
2822 self.native_json = True
2823 else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
2824
2825 - def LIKE(self,first,second):
2826 args = (self.expand(first), self.expand(second,'string'))
2827 if not first.type in ('string', 'text', 'json'):
2828 return '(%s LIKE %s)' % (
2829 self.CAST(args[0], 'CHAR(%s)' % first.length), args[1])
2830 else:
2831 return '(%s LIKE %s)' % args
2832
2833 - def ILIKE(self,first,second):
2834 args = (self.expand(first), self.expand(second,'string'))
2835 if not first.type in ('string', 'text', 'json'):
2836 return '(%s LIKE %s)' % (
2837 self.CAST(args[0], 'CHAR(%s)' % first.length), args[1])
2838 else:
2839 return '(%s ILIKE %s)' % args
2840
2841 - def REGEXP(self,first,second):
2842 return '(%s ~ %s)' % (self.expand(first),
2843 self.expand(second,'string'))
2844
2845 - def STARTSWITH(self,first,second):
2846 return '(%s ILIKE %s)' % (self.expand(first),
2847 self.expand(second+'%','string'))
2848
2849 - def ENDSWITH(self,first,second):
2850 return '(%s ILIKE %s)' % (self.expand(first),
2851 self.expand('%'+second,'string'))
2852
2853
2854
2855 - def ST_ASGEOJSON(self, first, second):
2856 """
2857 http://postgis.org/docs/ST_AsGeoJSON.html
2858 """
2859 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
2860 self.expand(first), second['precision'], second['options'])
2861
2862 - def ST_ASTEXT(self, first):
2863 """
2864 http://postgis.org/docs/ST_AsText.html
2865 """
2866 return 'ST_AsText(%s)' %(self.expand(first))
2867
2868 - def ST_X(self, first):
2869 """
2870 http://postgis.org/docs/ST_X.html
2871 """
2872 return 'ST_X(%s)' %(self.expand(first))
2873
2874 - def ST_Y(self, first):
2875 """
2876 http://postgis.org/docs/ST_Y.html
2877 """
2878 return 'ST_Y(%s)' %(self.expand(first))
2879
2880 - def ST_CONTAINS(self, first, second):
2881 """
2882 http://postgis.org/docs/ST_Contains.html
2883 """
2884 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2885
2886 - def ST_DISTANCE(self, first, second):
2887 """
2888 http://postgis.org/docs/ST_Distance.html
2889 """
2890 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2891
2892 - def ST_EQUALS(self, first, second):
2893 """
2894 http://postgis.org/docs/ST_Equals.html
2895 """
2896 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2897
2898 - def ST_INTERSECTS(self, first, second):
2899 """
2900 http://postgis.org/docs/ST_Intersects.html
2901 """
2902 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2903
2904 - def ST_OVERLAPS(self, first, second):
2905 """
2906 http://postgis.org/docs/ST_Overlaps.html
2907 """
2908 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2909
2910 - def ST_SIMPLIFY(self, first, second):
2911 """
2912 http://postgis.org/docs/ST_Simplify.html
2913 """
2914 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2915
2916 - def ST_TOUCHES(self, first, second):
2917 """
2918 http://postgis.org/docs/ST_Touches.html
2919 """
2920 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2921
2922 - def ST_WITHIN(self, first, second):
2923 """
2924 http://postgis.org/docs/ST_Within.html
2925 """
2926 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2927
2928 - def represent(self, obj, fieldtype):
2929 field_is_type = fieldtype.startswith
2930 if field_is_type('geo'):
2931 srid = 4326
2932 geotype, parms = fieldtype[:-1].split('(')
2933 parms = parms.split(',')
2934 if len(parms) >= 2:
2935 schema, srid = parms[:2]
2936 if field_is_type('geometry'):
2937 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2938 elif field_is_type('geography'):
2939 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
2940
2941
2942 return value
2943 return BaseAdapter.represent(self, obj, fieldtype)
2944
2945 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2946 drivers = ('psycopg2','pg8000')
2947
2948 types = {
2949 'boolean': 'CHAR(1)',
2950 'string': 'VARCHAR(%(length)s)',
2951 'text': 'TEXT',
2952 'json': 'TEXT',
2953 'password': 'VARCHAR(%(length)s)',
2954 'blob': 'BYTEA',
2955 'upload': 'VARCHAR(%(length)s)',
2956 'integer': 'INTEGER',
2957 'bigint': 'BIGINT',
2958 'float': 'FLOAT',
2959 'double': 'FLOAT8',
2960 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2961 'date': 'DATE',
2962 'time': 'TIME',
2963 'datetime': 'TIMESTAMP',
2964 'id': 'SERIAL PRIMARY KEY',
2965 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2966 'list:integer': 'BIGINT[]',
2967 'list:string': 'TEXT[]',
2968 'list:reference': 'BIGINT[]',
2969 'geometry': 'GEOMETRY',
2970 'geography': 'GEOGRAPHY',
2971 'big-id': 'BIGSERIAL PRIMARY KEY',
2972 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2973 }
2974
2975 - def parse_list_integers(self, value, field_type):
2977
2978 - def parse_list_references(self, value, field_type):
2979 return [self.parse_reference(r, field_type[5:]) for r in value]
2980
2981 - def parse_list_strings(self, value, field_type):
2983
2984 - def represent(self, obj, fieldtype):
2985 field_is_type = fieldtype.startswith
2986 if field_is_type('list:'):
2987 if not obj:
2988 obj = []
2989 elif not isinstance(obj, (list, tuple)):
2990 obj = [obj]
2991 if field_is_type('list:string'):
2992 obj = map(str,obj)
2993 else:
2994 obj = map(int,obj)
2995 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
2996 return BaseAdapter.represent(self, obj, fieldtype)
2997
2998
2999 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
3000 drivers = ('zxJDBC',)
3001
3002 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
3003
3004 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3005 credential_decoder=IDENTITY, driver_args={},
3006 adapter_args={}, do_connect=True, after_connection=None ):
3007 self.db = db
3008 self.dbengine = "postgres"
3009 self.uri = uri
3010 if do_connect: self.find_driver(adapter_args,uri)
3011 self.pool_size = pool_size
3012 self.folder = folder
3013 self.db_codec = db_codec
3014 self._after_connection = after_connection
3015 self.find_or_make_work_folder()
3016 ruri = uri.split('://',1)[1]
3017 m = self.REGEX_URI.match(ruri)
3018 if not m:
3019 raise SyntaxError("Invalid URI string in DAL")
3020 user = credential_decoder(m.group('user'))
3021 if not user:
3022 raise SyntaxError('User required')
3023 password = credential_decoder(m.group('password'))
3024 if not password:
3025 password = ''
3026 host = m.group('host')
3027 if not host:
3028 raise SyntaxError('Host name required')
3029 db = m.group('db')
3030 if not db:
3031 raise SyntaxError('Database name required')
3032 port = m.group('port') or '5432'
3033 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
3034 def connector(msg=msg,driver_args=driver_args):
3035 return self.driver.connect(*msg,**driver_args)
3036 self.connector = connector
3037 if do_connect: self.reconnect()
3038
3039 - def after_connection(self):
3040 self.connection.set_client_encoding('UTF8')
3041 self.execute('BEGIN;')
3042 self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
3043 self.try_json()
3044
3047 drivers = ('cx_Oracle',)
3048
3049 commit_on_alter_table = False
3050 types = {
3051 'boolean': 'CHAR(1)',
3052 'string': 'VARCHAR2(%(length)s)',
3053 'text': 'CLOB',
3054 'json': 'CLOB',
3055 'password': 'VARCHAR2(%(length)s)',
3056 'blob': 'CLOB',
3057 'upload': 'VARCHAR2(%(length)s)',
3058 'integer': 'INT',
3059 'bigint': 'NUMBER',
3060 'float': 'FLOAT',
3061 'double': 'BINARY_DOUBLE',
3062 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3063 'date': 'DATE',
3064 'time': 'CHAR(8)',
3065 'datetime': 'DATE',
3066 'id': 'NUMBER PRIMARY KEY',
3067 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3068 'list:integer': 'CLOB',
3069 'list:string': 'CLOB',
3070 'list:reference': 'CLOB',
3071 'big-id': 'NUMBER PRIMARY KEY',
3072 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3073 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3074 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3075 }
3076
3078 return '%s_sequence' % tablename
3079
3081 return '%s_trigger' % tablename
3082
3084 return 'LEFT OUTER JOIN'
3085
3087 return 'dbms_random.value'
3088
3089 - def NOT_NULL(self,default,field_type):
3090 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3091
3092 - def _drop(self,table,mode):
3095
3096 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3097 if limitby:
3098 (lmin, lmax) = limitby
3099 if len(sql_w) > 1:
3100 sql_w_row = sql_w + ' AND w_row > %i' % lmin
3101 else:
3102 sql_w_row = 'WHERE w_row > %i' % lmin
3103 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
3104 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3105
3111
3113 if fieldtype == 'blob':
3114 obj = base64.b64encode(str(obj))
3115 return ":CLOB('%s')" % obj
3116 elif fieldtype == 'date':
3117 if isinstance(obj, (datetime.date, datetime.datetime)):
3118 obj = obj.isoformat()[:10]
3119 else:
3120 obj = str(obj)
3121 return "to_date('%s','yyyy-mm-dd')" % obj
3122 elif fieldtype == 'datetime':
3123 if isinstance(obj, datetime.datetime):
3124 obj = obj.isoformat()[:19].replace('T',' ')
3125 elif isinstance(obj, datetime.date):
3126 obj = obj.isoformat()[:10]+' 00:00:00'
3127 else:
3128 obj = str(obj)
3129 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
3130 return None
3131
3132 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3133 credential_decoder=IDENTITY, driver_args={},
3134 adapter_args={}, do_connect=True, after_connection=None):
3135 self.db = db
3136 self.dbengine = "oracle"
3137 self.uri = uri
3138 if do_connect: self.find_driver(adapter_args,uri)
3139 self.pool_size = pool_size
3140 self.folder = folder
3141 self.db_codec = db_codec
3142 self._after_connection = after_connection
3143 self.find_or_make_work_folder()
3144 ruri = uri.split('://',1)[1]
3145 if not 'threaded' in driver_args:
3146 driver_args['threaded']=True
3147 def connector(uri=ruri,driver_args=driver_args):
3148 return self.driver.connect(uri,**driver_args)
3149 self.connector = connector
3150 if do_connect: self.reconnect()
3151
3153 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3154 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3155
3156 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
3157
3158 - def execute(self, command, args=None):
3159 args = args or []
3160 i = 1
3161 while True:
3162 m = self.oracle_fix.match(command)
3163 if not m:
3164 break
3165 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
3166 args.append(m.group('clob')[6:-2].replace("''", "'"))
3167 i += 1
3168 if command[-1:]==';':
3169 command = command[:-1]
3170 return self.log_execute(command, args)
3171
3173 tablename = table._tablename
3174 id_name = table._id.name
3175 sequence_name = table._sequence_name
3176 trigger_name = table._trigger_name
3177 self.execute(query)
3178 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
3179 self.execute("""
3180 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
3181 DECLARE
3182 curr_val NUMBER;
3183 diff_val NUMBER;
3184 PRAGMA autonomous_transaction;
3185 BEGIN
3186 IF :NEW.%(id)s IS NOT NULL THEN
3187 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3188 diff_val := :NEW.%(id)s - curr_val - 1;
3189 IF diff_val != 0 THEN
3190 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
3191 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3192 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
3193 END IF;
3194 END IF;
3195 SELECT %(sequence_name)s.nextval INTO :NEW.%(id)s FROM DUAL;
3196 END;
3197 """ % dict(trigger_name=trigger_name, tablename=tablename,
3198 sequence_name=sequence_name,id=id_name))
3199
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3215 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description):
3216 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
3217 for c in r]) for r in self.cursor]
3218 else:
3219 return self.cursor.fetchall()
3220
3222 drivers = ('pyodbc',)
3223 T_SEP = 'T'
3224
3225 QUOTE_TEMPLATE = "[%s]"
3226
3227 types = {
3228 'boolean': 'BIT',
3229 'string': 'VARCHAR(%(length)s)',
3230 'text': 'TEXT',
3231 'json': 'TEXT',
3232 'password': 'VARCHAR(%(length)s)',
3233 'blob': 'IMAGE',
3234 'upload': 'VARCHAR(%(length)s)',
3235 'integer': 'INT',
3236 'bigint': 'BIGINT',
3237 'float': 'FLOAT',
3238 'double': 'FLOAT',
3239 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3240 'date': 'DATETIME',
3241 'time': 'CHAR(8)',
3242 'datetime': 'DATETIME',
3243 'id': 'INT IDENTITY PRIMARY KEY',
3244 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3245 'list:integer': 'TEXT',
3246 'list:string': 'TEXT',
3247 'list:reference': 'TEXT',
3248 'geometry': 'geometry',
3249 'geography': 'geography',
3250 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3251 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3252 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3253 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3254 }
3255
3257 return '; ALTER TABLE %s ADD ' % tablename
3258
3261
3263 return "DATEPART(%s,%s)" % (what, self.expand(field))
3264
3266 return 'LEFT OUTER JOIN'
3267
3270
3273
3274 - def CAST(self, first, second):
3276
3278 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
3279
3281 return 'PRIMARY KEY CLUSTERED (%s)' % key
3282
3284 if what == 'LENGTH':
3285 what = 'LEN'
3286 return "%s(%s)" % (what, self.expand(first))
3287
3288
3289 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3290 if limitby:
3291 (lmin, lmax) = limitby
3292 sql_s += ' TOP %i' % lmax
3293 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3294
3295 TRUE = 1
3296 FALSE = 0
3297
3298 REGEX_DSN = re.compile('^(?P<dsn>.+)$')
3299 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
3300 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
3301
3302 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3303 credential_decoder=IDENTITY, driver_args={},
3304 adapter_args={}, do_connect=True, srid=4326,
3305 after_connection=None):
3306 self.db = db
3307 self.dbengine = "mssql"
3308 self.uri = uri
3309 if do_connect: self.find_driver(adapter_args,uri)
3310 self.pool_size = pool_size
3311 self.folder = folder
3312 self.db_codec = db_codec
3313 self._after_connection = after_connection
3314 self.srid = srid
3315 self.find_or_make_work_folder()
3316
3317 ruri = uri.split('://',1)[1]
3318 if '@' not in ruri:
3319 try:
3320 m = self.REGEX_DSN.match(ruri)
3321 if not m:
3322 raise SyntaxError(
3323 'Parsing uri string(%s) has no result' % self.uri)
3324 dsn = m.group('dsn')
3325 if not dsn:
3326 raise SyntaxError('DSN required')
3327 except SyntaxError:
3328 e = sys.exc_info()[1]
3329 LOGGER.error('NdGpatch error')
3330 raise e
3331
3332 cnxn = dsn
3333 else:
3334 m = self.REGEX_URI.match(ruri)
3335 if not m:
3336 raise SyntaxError(
3337 "Invalid URI string in DAL: %s" % self.uri)
3338 user = credential_decoder(m.group('user'))
3339 if not user:
3340 raise SyntaxError('User required')
3341 password = credential_decoder(m.group('password'))
3342 if not password:
3343 password = ''
3344 host = m.group('host')
3345 if not host:
3346 raise SyntaxError('Host name required')
3347 db = m.group('db')
3348 if not db:
3349 raise SyntaxError('Database name required')
3350 port = m.group('port') or '1433'
3351
3352
3353
3354 argsdict = { 'DRIVER':'{SQL Server}' }
3355 urlargs = m.group('urlargs') or ''
3356 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
3357 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
3358 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
3359 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
3360 % (host, port, db, user, password, urlargs)
3361 def connector(cnxn=cnxn,driver_args=driver_args):
3362 return self.driver.connect(cnxn,**driver_args)
3363 self.connector = connector
3364 if do_connect: self.reconnect()
3365
3367
3368 self.execute('SELECT SCOPE_IDENTITY();')
3369 return long(self.cursor.fetchone()[0])
3370
3371 - def rowslice(self,rows,minimum=0,maximum=None):
3372 if maximum is None:
3373 return rows[minimum:]
3374 return rows[minimum:maximum]
3375
3376 - def EPOCH(self, first):
3377 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3378
3381
3382
3383
3384
3385
3386 - def ST_ASTEXT(self, first):
3387 return '%s.STAsText()' %(self.expand(first))
3388
3391
3394
3397
3400
3403
3404
3405
3408
3411
3413 field_is_type = fieldtype.startswith
3414 if field_is_type('geometry'):
3415 srid = 0
3416 geotype, parms = fieldtype[:-1].split('(')
3417 if parms:
3418 srid = parms
3419 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3420 elif fieldtype == 'geography':
3421 srid = 4326
3422 geotype, parms = fieldtype[:-1].split('(')
3423 if parms:
3424 srid = parms
3425 return "geography::STGeomFromText('%s',%s)" %(obj, srid)
3426
3427
3428 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3429 return BaseAdapter.represent(self, obj, fieldtype)
3430
3433 """ experimental support for pagination in MSSQL"""
3434 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3435 if limitby:
3436 (lmin, lmax) = limitby
3437 if lmin == 0:
3438 sql_s += ' TOP %i' % lmax
3439 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3440 lmin += 1
3441 sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
3442 sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
3443 sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
3444 sql_f_inner = [f for f in sql_f.split(',')]
3445 sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
3446 sql_f_iproxy = ', '.join(sql_f_iproxy)
3447 sql_f_oproxy = ', '.join(sql_f_outer)
3448 return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
3449 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3450 - def rowslice(self,rows,minimum=0,maximum=None):
3452
3454 """ support for true pagination in MSSQL >= 2012"""
3455
3456 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3457 if limitby:
3458 (lmin, lmax) = limitby
3459 if lmin == 0:
3460
3461
3462
3463 sql_s += ' TOP %i' % lmax
3464 else:
3465 if not sql_o:
3466
3467
3468 sql_o += ' ORDER BY %s' % self.RANDOM()
3469 sql_o += ' OFFSET %i ROWS FETCH NEXT %i ROWS ONLY' % (lmin, lmax - lmin)
3470 return 'SELECT %s %s FROM %s%s%s;' % \
3471 (sql_s, sql_f, sql_t, sql_w, sql_o)
3472
3473 - def rowslice(self,rows,minimum=0,maximum=None):
3475
3477 drivers = ('pyodbc',)
3478
3479 types = {
3480 'boolean': 'CHAR(1)',
3481 'string': 'NVARCHAR(%(length)s)',
3482 'text': 'NTEXT',
3483 'json': 'NTEXT',
3484 'password': 'NVARCHAR(%(length)s)',
3485 'blob': 'IMAGE',
3486 'upload': 'NVARCHAR(%(length)s)',
3487 'integer': 'INT',
3488 'bigint': 'BIGINT',
3489 'float': 'FLOAT',
3490 'double': 'FLOAT',
3491 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3492 'date': 'DATETIME',
3493 'time': 'CHAR(8)',
3494 'datetime': 'DATETIME',
3495 'id': 'INT IDENTITY PRIMARY KEY',
3496 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3497 'list:integer': 'NTEXT',
3498 'list:string': 'NTEXT',
3499 'list:reference': 'NTEXT',
3500 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3501 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3502 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3503 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3504 }
3505
3507 value = BaseAdapter.represent(self, obj, fieldtype)
3508 if fieldtype in ('string','text', 'json') and value[:1]=="'":
3509 value = 'N'+value
3510 return value
3511
3514
3516 drivers = ('pyodbc',)
3517 T_SEP = ' '
3518
3519 types = {
3520 'boolean': 'BOOLEAN',
3521 'string': 'VARCHAR(%(length)s)',
3522 'text': 'BYTEA',
3523 'json': 'VARCHAR(%(length)s)',
3524 'password': 'VARCHAR(%(length)s)',
3525 'blob': 'BYTEA',
3526 'upload': 'VARCHAR(%(length)s)',
3527 'integer': 'INT',
3528 'bigint': 'BIGINT',
3529 'float': 'FLOAT',
3530 'double': 'DOUBLE PRECISION',
3531 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3532 'date': 'DATE',
3533 'time': 'TIME',
3534 'datetime': 'DATETIME',
3535 'id': 'IDENTITY',
3536 'reference': 'INT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3537 'list:integer': 'BYTEA',
3538 'list:string': 'BYTEA',
3539 'list:reference': 'BYTEA',
3540 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3541 }
3542
3543
3545 return "DATE_PART('%s', TIMESTAMP %s)" % (what, self.expand(first))
3546
3548 tablename = table._tablename
3549 return ['TRUNCATE %s %s;' % (tablename, mode or '')]
3550
3551 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3552 if limitby:
3553 (lmin, lmax) = limitby
3554 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
3555 return 'SELECT %s %s FROM %s%s%s;' % \
3556 (sql_s, sql_f, sql_t, sql_w, sql_o)
3557
3559 self.execute('SELECT LAST_INSERT_ID();')
3560 return long(self.cursor.fetchone()[0])
3561
3564
3566 drivers = ('Sybase',)
3567
3568 types = {
3569 'boolean': 'BIT',
3570 'string': 'CHAR VARYING(%(length)s)',
3571 'text': 'TEXT',
3572 'json': 'TEXT',
3573 'password': 'CHAR VARYING(%(length)s)',
3574 'blob': 'IMAGE',
3575 'upload': 'CHAR VARYING(%(length)s)',
3576 'integer': 'INT',
3577 'bigint': 'BIGINT',
3578 'float': 'FLOAT',
3579 'double': 'FLOAT',
3580 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3581 'date': 'DATETIME',
3582 'time': 'CHAR(8)',
3583 'datetime': 'DATETIME',
3584 'id': 'INT IDENTITY PRIMARY KEY',
3585 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3586 'list:integer': 'TEXT',
3587 'list:string': 'TEXT',
3588 'list:reference': 'TEXT',
3589 'geometry': 'geometry',
3590 'geography': 'geography',
3591 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3592 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3593 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3594 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3595 }
3596
3597
3598 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3599 credential_decoder=IDENTITY, driver_args={},
3600 adapter_args={}, do_connect=True, srid=4326,
3601 after_connection=None):
3602 self.db = db
3603 self.dbengine = "sybase"
3604 self.uri = uri
3605 if do_connect: self.find_driver(adapter_args,uri)
3606 self.pool_size = pool_size
3607 self.folder = folder
3608 self.db_codec = db_codec
3609 self._after_connection = after_connection
3610 self.srid = srid
3611 self.find_or_make_work_folder()
3612
3613 ruri = uri.split('://',1)[1]
3614 if '@' not in ruri:
3615 try:
3616 m = self.REGEX_DSN.match(ruri)
3617 if not m:
3618 raise SyntaxError(
3619 'Parsing uri string(%s) has no result' % self.uri)
3620 dsn = m.group('dsn')
3621 if not dsn:
3622 raise SyntaxError('DSN required')
3623 except SyntaxError:
3624 e = sys.exc_info()[1]
3625 LOGGER.error('NdGpatch error')
3626 raise e
3627 else:
3628 m = self.REGEX_URI.match(uri)
3629 if not m:
3630 raise SyntaxError(
3631 "Invalid URI string in DAL: %s" % self.uri)
3632 user = credential_decoder(m.group('user'))
3633 if not user:
3634 raise SyntaxError('User required')
3635 password = credential_decoder(m.group('password'))
3636 if not password:
3637 password = ''
3638 host = m.group('host')
3639 if not host:
3640 raise SyntaxError('Host name required')
3641 db = m.group('db')
3642 if not db:
3643 raise SyntaxError('Database name required')
3644 port = m.group('port') or '1433'
3645
3646 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
3647
3648 driver_args.update(user = credential_decoder(user),
3649 password = credential_decoder(password))
3650
3651 def connector(dsn=dsn,driver_args=driver_args):
3652 return self.driver.connect(dsn,**driver_args)
3653 self.connector = connector
3654 if do_connect: self.reconnect()
3655
3658 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3659
3660 commit_on_alter_table = False
3661 support_distributed_transaction = True
3662 types = {
3663 'boolean': 'CHAR(1)',
3664 'string': 'VARCHAR(%(length)s)',
3665 'text': 'BLOB SUB_TYPE 1',
3666 'json': 'BLOB SUB_TYPE 1',
3667 'password': 'VARCHAR(%(length)s)',
3668 'blob': 'BLOB SUB_TYPE 0',
3669 'upload': 'VARCHAR(%(length)s)',
3670 'integer': 'INTEGER',
3671 'bigint': 'BIGINT',
3672 'float': 'FLOAT',
3673 'double': 'DOUBLE PRECISION',
3674 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3675 'date': 'DATE',
3676 'time': 'TIME',
3677 'datetime': 'TIMESTAMP',
3678 'id': 'INTEGER PRIMARY KEY',
3679 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3680 'list:integer': 'BLOB SUB_TYPE 1',
3681 'list:string': 'BLOB SUB_TYPE 1',
3682 'list:reference': 'BLOB SUB_TYPE 1',
3683 'big-id': 'BIGINT PRIMARY KEY',
3684 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3685 }
3686
3688 return 'genid_%s' % tablename
3689
3691 return 'trg_id_%s' % tablename
3692
3695
3696 - def EPOCH(self, first):
3697 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3698
3699 - def NOT_NULL(self,default,field_type):
3700 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3701
3703 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3704
3707
3708 - def CONTAINS(self,first,second,case_sensitive=False):
3714
3715 - def _drop(self,table,mode):
3718
3719 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3720 if limitby:
3721 (lmin, lmax) = limitby
3722 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
3723 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3724
3726 return ['DELETE FROM %s;' % table._tablename,
3727 'SET GENERATOR %s TO 0;' % table._sequence_name]
3728
3729 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
3730
3731 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3732 credential_decoder=IDENTITY, driver_args={},
3733 adapter_args={}, do_connect=True, after_connection=None):
3734 self.db = db
3735 self.dbengine = "firebird"
3736 self.uri = uri
3737 if do_connect: self.find_driver(adapter_args,uri)
3738 self.pool_size = pool_size
3739 self.folder = folder
3740 self.db_codec = db_codec
3741 self._after_connection = after_connection
3742 self.find_or_make_work_folder()
3743 ruri = uri.split('://',1)[1]
3744 m = self.REGEX_URI.match(ruri)
3745 if not m:
3746 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
3747 user = credential_decoder(m.group('user'))
3748 if not user:
3749 raise SyntaxError('User required')
3750 password = credential_decoder(m.group('password'))
3751 if not password:
3752 password = ''
3753 host = m.group('host')
3754 if not host:
3755 raise SyntaxError('Host name required')
3756 port = int(m.group('port') or 3050)
3757 db = m.group('db')
3758 if not db:
3759 raise SyntaxError('Database name required')
3760 charset = m.group('charset') or 'UTF8'
3761 driver_args.update(dsn='%s/%s:%s' % (host,port,db),
3762 user = credential_decoder(user),
3763 password = credential_decoder(password),
3764 charset = charset)
3765
3766 def connector(driver_args=driver_args):
3767 return self.driver.connect(**driver_args)
3768 self.connector = connector
3769 if do_connect: self.reconnect()
3770
3779
3784
3787 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3788
3789 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
3790
3791 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3792 credential_decoder=IDENTITY, driver_args={},
3793 adapter_args={}, do_connect=True, after_connection=None):
3794 self.db = db
3795 self.dbengine = "firebird"
3796 self.uri = uri
3797 if do_connect: self.find_driver(adapter_args,uri)
3798 self.pool_size = pool_size
3799 self.folder = folder
3800 self.db_codec = db_codec
3801 self._after_connection = after_connection
3802 self.find_or_make_work_folder()
3803 ruri = uri.split('://',1)[1]
3804 m = self.REGEX_URI.match(ruri)
3805 if not m:
3806 raise SyntaxError(
3807 "Invalid URI string in DAL: %s" % self.uri)
3808 user = credential_decoder(m.group('user'))
3809 if not user:
3810 raise SyntaxError('User required')
3811 password = credential_decoder(m.group('password'))
3812 if not password:
3813 password = ''
3814 pathdb = m.group('path')
3815 if not pathdb:
3816 raise SyntaxError('Path required')
3817 charset = m.group('charset')
3818 if not charset:
3819 charset = 'UTF8'
3820 host = ''
3821 driver_args.update(host=host,
3822 database=pathdb,
3823 user=credential_decoder(user),
3824 password=credential_decoder(password),
3825 charset=charset)
3826
3827 def connector(driver_args=driver_args):
3828 return self.driver.connect(**driver_args)
3829 self.connector = connector
3830 if do_connect: self.reconnect()
3831
3937
3942
3945
3957
3959 drivers = ('pyodbc',)
3960
3961 types = {
3962 'boolean': 'CHAR(1)',
3963 'string': 'VARCHAR(%(length)s)',
3964 'text': 'CLOB',
3965 'json': 'CLOB',
3966 'password': 'VARCHAR(%(length)s)',
3967 'blob': 'BLOB',
3968 'upload': 'VARCHAR(%(length)s)',
3969 'integer': 'INT',
3970 'bigint': 'BIGINT',
3971 'float': 'REAL',
3972 'double': 'DOUBLE',
3973 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3974 'date': 'DATE',
3975 'time': 'TIME',
3976 'datetime': 'TIMESTAMP',
3977 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3978 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3979 'list:integer': 'CLOB',
3980 'list:string': 'CLOB',
3981 'list:reference': 'CLOB',
3982 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3983 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3984 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3985 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3986 }
3987
3989 return 'LEFT OUTER JOIN'
3990
3993
3994 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3995 if limitby:
3996 (lmin, lmax) = limitby
3997 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
3998 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3999
4001 if fieldtype == 'blob':
4002 obj = base64.b64encode(str(obj))
4003 return "BLOB('%s')" % obj
4004 elif fieldtype == 'datetime':
4005 if isinstance(obj, datetime.datetime):
4006 obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
4007 elif isinstance(obj, datetime.date):
4008 obj = obj.isoformat()[:10]+'-00.00.00'
4009 return "'%s'" % obj
4010 return None
4011
4012 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4013 credential_decoder=IDENTITY, driver_args={},
4014 adapter_args={}, do_connect=True, after_connection=None):
4027 self.connector = connector
4028 if do_connect: self.reconnect()
4029
4031 if command[-1:]==';':
4032 command = command[:-1]
4033 return self.log_execute(command)
4034
4036 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
4037 return long(self.cursor.fetchone()[0])
4038
4039 - def rowslice(self,rows,minimum=0,maximum=None):
4040 if maximum is None:
4041 return rows[minimum:]
4042 return rows[minimum:maximum]
4043
4046 drivers = ('pyodbc',)
4047
4048 types = {
4049 'boolean': 'CHAR(1)',
4050 'string': 'VARCHAR(%(length)s)',
4051 'text': 'VARCHAR(2000)',
4052 'json': 'VARCHAR(4000)',
4053 'password': 'VARCHAR(%(length)s)',
4054 'blob': 'BLOB',
4055 'upload': 'VARCHAR(%(length)s)',
4056 'integer': 'INT',
4057 'bigint': 'BIGINT',
4058 'float': 'REAL',
4059 'double': 'DOUBLE',
4060 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4061 'date': 'DATE',
4062 'time': 'TIME',
4063 'datetime': 'TIMESTAMP',
4064
4065
4066 'id': 'INT GENERATED ALWAYS AS IDENTITY',
4067 'reference': 'INT',
4068 'list:integer': 'VARCHAR(4000)',
4069 'list:string': 'VARCHAR(4000)',
4070 'list:reference': 'VARCHAR(4000)',
4071 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',
4072 'big-reference': 'BIGINT',
4073 'reference FK': ' REFERENCES %(foreign_key)s',
4074 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
4075 }
4076
4077 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4078 credential_decoder=IDENTITY, driver_args={},
4079 adapter_args={}, do_connect=True, after_connection=None):
4092 self.connector = connector
4093 if do_connect: self.reconnect()
4094
4095 - def close(self,action='commit',really=True):
4100
4102 return 'LEFT OUTER JOIN'
4103
4104
4105 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4106 if limitby:
4107 (lmin, lmax) = limitby
4108 sql_s += ' TOP %i' % lmax
4109 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4110
4112 tablename = table._tablename
4113 return ['DELETE FROM %s ALL;' % (tablename)]
4114
4115 INGRES_SEQNAME='ii***lineitemsequence'
4120 drivers = ('pyodbc',)
4121
4122 types = {
4123 'boolean': 'CHAR(1)',
4124 'string': 'VARCHAR(%(length)s)',
4125 'text': 'CLOB',
4126 'json': 'CLOB',
4127 'password': 'VARCHAR(%(length)s)',
4128 'blob': 'BLOB',
4129 'upload': 'VARCHAR(%(length)s)',
4130 'integer': 'INTEGER4',
4131 'bigint': 'BIGINT',
4132 'float': 'FLOAT',
4133 'double': 'FLOAT8',
4134 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4135 'date': 'ANSIDATE',
4136 'time': 'TIME WITHOUT TIME ZONE',
4137 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
4138 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
4139 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4140 'list:integer': 'CLOB',
4141 'list:string': 'CLOB',
4142 'list:reference': 'CLOB',
4143 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
4144 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4145 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4146 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4147 }
4148
4150 return 'LEFT OUTER JOIN'
4151
4154
4155 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4156 if limitby:
4157 (lmin, lmax) = limitby
4158 fetch_amt = lmax - lmin
4159 if fetch_amt:
4160 sql_s += ' FIRST %d ' % (fetch_amt, )
4161 if lmin:
4162
4163 sql_o += ' OFFSET %d' % (lmin, )
4164 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4165
4166 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4167 credential_decoder=IDENTITY, driver_args={},
4168 adapter_args={}, do_connect=True, after_connection=None):
4169 self.db = db
4170 self.dbengine = "ingres"
4171 self._driver = pyodbc
4172 self.uri = uri
4173 if do_connect: self.find_driver(adapter_args,uri)
4174 self.pool_size = pool_size
4175 self.folder = folder
4176 self.db_codec = db_codec
4177 self._after_connection = after_connection
4178 self.find_or_make_work_folder()
4179 connstr = uri.split(':', 1)[1]
4180
4181 connstr = connstr.lstrip()
4182 while connstr.startswith('/'):
4183 connstr = connstr[1:]
4184 if '=' in connstr:
4185
4186 ruri = connstr
4187 else:
4188
4189 database_name = connstr
4190 default_driver_name = 'Ingres'
4191 vnode = '(local)'
4192 servertype = 'ingres'
4193 ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name)
4194 def connector(cnxn=ruri,driver_args=driver_args):
4195 return self.driver.connect(cnxn,**driver_args)
4196
4197 self.connector = connector
4198
4199
4200 if do_connect: self.reconnect()
4201
4203
4204
4205
4206 if hasattr(table,'_primarykey'):
4207 modify_tbl_sql = 'modify %s to btree unique on %s' % \
4208 (table._tablename,
4209 ', '.join(["'%s'" % x for x in table.primarykey]))
4210 self.execute(modify_tbl_sql)
4211 else:
4212 tmp_seqname='%s_iisq' % table._tablename
4213 query=query.replace(INGRES_SEQNAME, tmp_seqname)
4214 self.execute('create sequence %s' % tmp_seqname)
4215 self.execute(query)
4216 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
4217
4218
4220 tmp_seqname='%s_iisq' % table
4221 self.execute('select current value for %s' % tmp_seqname)
4222 return long(self.cursor.fetchone()[0])
4223
4226
4227 drivers = ('pyodbc',)
4228
4229 types = {
4230 'boolean': 'CHAR(1)',
4231 'string': 'NVARCHAR(%(length)s)',
4232 'text': 'NCLOB',
4233 'json': 'NCLOB',
4234 'password': 'NVARCHAR(%(length)s)',
4235 'blob': 'BLOB',
4236 'upload': 'VARCHAR(%(length)s)',
4237 'integer': 'INTEGER4',
4238 'bigint': 'BIGINT',
4239 'float': 'FLOAT',
4240 'double': 'FLOAT8',
4241 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4242 'date': 'ANSIDATE',
4243 'time': 'TIME WITHOUT TIME ZONE',
4244 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
4245 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
4246 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4247 'list:integer': 'NCLOB',
4248 'list:string': 'NCLOB',
4249 'list:reference': 'NCLOB',
4250 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
4251 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4252 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4253 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4254 }
4255
4257 drivers = ('sapdb',)
4258
4259 support_distributed_transaction = False
4260 types = {
4261 'boolean': 'CHAR(1)',
4262 'string': 'VARCHAR(%(length)s)',
4263 'text': 'LONG',
4264 'json': 'LONG',
4265 'password': 'VARCHAR(%(length)s)',
4266 'blob': 'LONG',
4267 'upload': 'VARCHAR(%(length)s)',
4268 'integer': 'INT',
4269 'bigint': 'BIGINT',
4270 'float': 'FLOAT',
4271 'double': 'DOUBLE PRECISION',
4272 'decimal': 'FIXED(%(precision)s,%(scale)s)',
4273 'date': 'DATE',
4274 'time': 'TIME',
4275 'datetime': 'TIMESTAMP',
4276 'id': 'INT PRIMARY KEY',
4277 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4278 'list:integer': 'LONG',
4279 'list:string': 'LONG',
4280 'list:reference': 'LONG',
4281 'big-id': 'BIGINT PRIMARY KEY',
4282 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4283 }
4284
4286 return '%s_id_Seq' % table
4287
4288 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4289 if limitby:
4290 (lmin, lmax) = limitby
4291 if len(sql_w) > 1:
4292 sql_w_row = sql_w + ' AND w_row > %i' % lmin
4293 else:
4294 sql_w_row = 'WHERE w_row > %i' % lmin
4295 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
4296 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4297
4299
4300 self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
4301 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
4302 % (table._tablename, table._id.name, table._sequence_name))
4303 self.execute(query)
4304
4305 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
4306
4307
4308 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4309 credential_decoder=IDENTITY, driver_args={},
4310 adapter_args={}, do_connect=True, after_connection=None):
4311 self.db = db
4312 self.dbengine = "sapdb"
4313 self.uri = uri
4314 if do_connect: self.find_driver(adapter_args,uri)
4315 self.pool_size = pool_size
4316 self.folder = folder
4317 self.db_codec = db_codec
4318 self._after_connection = after_connection
4319 self.find_or_make_work_folder()
4320 ruri = uri.split('://',1)[1]
4321 m = self.REGEX_URI.match(ruri)
4322 if not m:
4323 raise SyntaxError("Invalid URI string in DAL")
4324 user = credential_decoder(m.group('user'))
4325 if not user:
4326 raise SyntaxError('User required')
4327 password = credential_decoder(m.group('password'))
4328 if not password:
4329 password = ''
4330 host = m.group('host')
4331 if not host:
4332 raise SyntaxError('Host name required')
4333 db = m.group('db')
4334 if not db:
4335 raise SyntaxError('Database name required')
4336 def connector(user=user, password=password, database=db,
4337 host=host, driver_args=driver_args):
4338 return self.driver.Connection(user, password, database,
4339 host, **driver_args)
4340 self.connector = connector
4341 if do_connect: self.reconnect()
4342
4344 self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
4345 return long(self.cursor.fetchone()[0])
4346
4348 drivers = ('cubriddb',)
4349
4350 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
4351
4352 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
4353 credential_decoder=IDENTITY, driver_args={},
4354 adapter_args={}, do_connect=True, after_connection=None):
4355 self.db = db
4356 self.dbengine = "cubrid"
4357 self.uri = uri
4358 if do_connect: self.find_driver(adapter_args,uri)
4359 self.pool_size = pool_size
4360 self.folder = folder
4361 self.db_codec = db_codec
4362 self._after_connection = after_connection
4363 self.find_or_make_work_folder()
4364 ruri = uri.split('://',1)[1]
4365 m = self.REGEX_URI.match(ruri)
4366 if not m:
4367 raise SyntaxError(
4368 "Invalid URI string in DAL: %s" % self.uri)
4369 user = credential_decoder(m.group('user'))
4370 if not user:
4371 raise SyntaxError('User required')
4372 password = credential_decoder(m.group('password'))
4373 if not password:
4374 password = ''
4375 host = m.group('host')
4376 if not host:
4377 raise SyntaxError('Host name required')
4378 db = m.group('db')
4379 if not db:
4380 raise SyntaxError('Database name required')
4381 port = int(m.group('port') or '30000')
4382 charset = m.group('charset') or 'utf8'
4383 user = credential_decoder(user)
4384 passwd = credential_decoder(password)
4385 def connector(host=host,port=port,db=db,
4386 user=user,passwd=password,driver_args=driver_args):
4387 return self.driver.connect(host,port,db,user,passwd,**driver_args)
4388 self.connector = connector
4389 if do_connect: self.reconnect()
4390
4392 self.execute('SET FOREIGN_KEY_CHECKS=1;')
4393 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4394
4399
4400 web2py_filesystem = False
4401
4403 return self.db._adapter.escape(obj)
4404
4406 if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'):
4407 raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now")
4408 self.db = db
4409 self.filename = filename
4410 self.mode = mode
4411 if not self.web2py_filesystem:
4412 if db._adapter.dbengine == 'mysql':
4413 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
4414 elif db._adapter.dbengine in ('postgres', 'sqlite'):
4415 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
4416 self.db.executesql(sql)
4417 DatabaseStoredFile.web2py_filesystem = True
4418 self.p=0
4419 self.data = ''
4420 if mode in ('r','rw','a'):
4421 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
4422 % filename
4423 rows = self.db.executesql(query)
4424 if rows:
4425 self.data = rows[0][0]
4426 elif exists(filename):
4427 datafile = open(filename, 'r')
4428 try:
4429 self.data = datafile.read()
4430 finally:
4431 datafile.close()
4432 elif mode in ('r','rw'):
4433 raise RuntimeError("File %s does not exist" % filename)
4434
4435 - def read(self, bytes):
4436 data = self.data[self.p:self.p+bytes]
4437 self.p += len(data)
4438 return data
4439
4441 i = self.data.find('\n',self.p)+1
4442 if i>0:
4443 data, self.p = self.data[self.p:i], i
4444 else:
4445 data, self.p = self.data[self.p:], len(self.data)
4446 return data
4447
4450
4452 if self.db is not None:
4453 self.db.executesql(
4454 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
4455 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
4456 % (self.filename, self.data.replace("'","''"))
4457 self.db.executesql(query)
4458 self.db.commit()
4459 self.db = None
4460
4463
4464 @staticmethod
4466 if exists(filename):
4467 return True
4468 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
4469 try:
4470 if db.executesql(query):
4471 return True
4472 except Exception, e:
4473 if not (db._adapter.isOperationalError(e) or
4474 db._adapter.isProgrammingError(e)):
4475 raise
4476
4477 tb = traceback.format_exc()
4478 LOGGER.error("Could not retrieve %s\n%s" % (filename, tb))
4479 return False
4480
4483
4486
4487 - def file_open(self, filename, mode='rb', lock=True):
4489
4492
4494 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
4495 self.db.executesql(query)
4496 self.db.commit()
4497
4499 uploads_in_blob = True
4500
4501 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
4502
4503 - def __init__(self, db, uri='google:sql://realm:domain/database',
4504 pool_size=0, folder=None, db_codec='UTF-8',
4505 credential_decoder=IDENTITY, driver_args={},
4506 adapter_args={}, do_connect=True, after_connection=None):
4507
4508 self.db = db
4509 self.dbengine = "mysql"
4510 self.uri = uri
4511 self.pool_size = pool_size
4512 self.db_codec = db_codec
4513 self._after_connection = after_connection
4514 if do_connect: self.find_driver(adapter_args, uri)
4515 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
4516 os.sep+'applications'+os.sep,1)[1])
4517 ruri = uri.split("://")[1]
4518 m = self.REGEX_URI.match(ruri)
4519 if not m:
4520 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
4521 instance = credential_decoder(m.group('instance'))
4522 self.dbstring = db = credential_decoder(m.group('db'))
4523 driver_args['instance'] = instance
4524 if not 'charset' in driver_args:
4525 driver_args['charset'] = 'utf8'
4526 self.createdb = createdb = adapter_args.get('createdb',True)
4527 if not createdb:
4528 driver_args['database'] = db
4529 def connector(driver_args=driver_args):
4530 return rdbms.connect(**driver_args)
4531 self.connector = connector
4532 if do_connect: self.reconnect()
4533
4535 if self.createdb:
4536
4537 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
4538 self.execute('USE %s' % self.dbstring)
4539 self.execute("SET FOREIGN_KEY_CHECKS=1;")
4540 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4541
4542 - def execute(self, command, *a, **b):
4544
4546 self.adapter_args = adapter_args
4547 self.driver = "google"
4548
4550 can_select_for_update = False
4551
4552 @staticmethod
4554 if isinstance(obj, str):
4555 return obj.decode('utf8')
4556 elif not isinstance(obj, unicode):
4557 return unicode(obj)
4558 return obj
4559
4561 return table._id > 0
4562
4564 field_is_type = fieldtype.startswith
4565 if isinstance(obj, CALLABLETYPES):
4566 obj = obj()
4567 if isinstance(fieldtype, SQLCustomType):
4568 return fieldtype.encoder(obj)
4569 if isinstance(obj, (Expression, Field)):
4570 raise SyntaxError("non supported on GAE")
4571 if self.dbengine == 'google:datastore':
4572 if isinstance(fieldtype, gae.Property):
4573 return obj
4574 is_string = isinstance(fieldtype,str)
4575 is_list = is_string and field_is_type('list:')
4576 if is_list:
4577 if not obj:
4578 obj = []
4579 if not isinstance(obj, (list, tuple)):
4580 obj = [obj]
4581 if obj == '' and not \
4582 (is_string and fieldtype[:2] in ['st','te', 'pa','up']):
4583 return None
4584 if not obj is None:
4585 if isinstance(obj, list) and not is_list:
4586 obj = [self.represent(o, fieldtype) for o in obj]
4587 elif fieldtype in ('integer','bigint','id'):
4588 obj = long(obj)
4589 elif fieldtype == 'double':
4590 obj = float(obj)
4591 elif is_string and field_is_type('reference'):
4592 if isinstance(obj, (Row, Reference)):
4593 obj = obj['id']
4594 obj = long(obj)
4595 elif fieldtype == 'boolean':
4596 if obj and not str(obj)[0].upper() in '0F':
4597 obj = True
4598 else:
4599 obj = False
4600 elif fieldtype == 'date':
4601 if not isinstance(obj, datetime.date):
4602 (y, m, d) = map(int,str(obj).strip().split('-'))
4603 obj = datetime.date(y, m, d)
4604 elif isinstance(obj,datetime.datetime):
4605 (y, m, d) = (obj.year, obj.month, obj.day)
4606 obj = datetime.date(y, m, d)
4607 elif fieldtype == 'time':
4608 if not isinstance(obj, datetime.time):
4609 time_items = map(int,str(obj).strip().split(':')[:3])
4610 if len(time_items) == 3:
4611 (h, mi, s) = time_items
4612 else:
4613 (h, mi, s) = time_items + [0]
4614 obj = datetime.time(h, mi, s)
4615 elif fieldtype == 'datetime':
4616 if not isinstance(obj, datetime.datetime):
4617 (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
4618 time_items = map(int,str(obj)[11:].strip().split(':')[:3])
4619 while len(time_items)<3:
4620 time_items.append(0)
4621 (h, mi, s) = time_items
4622 obj = datetime.datetime(y, m, d, h, mi, s)
4623 elif fieldtype == 'blob':
4624 pass
4625 elif fieldtype == 'json':
4626 if isinstance(obj, basestring):
4627 obj = self.to_unicode(obj)
4628 if have_serializers:
4629 obj = serializers.loads_json(obj)
4630 elif simplejson:
4631 obj = simplejson.loads(obj)
4632 else:
4633 raise RuntimeError("missing simplejson")
4634 elif is_string and field_is_type('list:string'):
4635 return map(self.to_unicode,obj)
4636 elif is_list:
4637 return map(int,obj)
4638 else:
4639 obj = self.to_unicode(obj)
4640 return obj
4641
4643 return 'insert %s in %s' % (fields, table)
4644
4645 - def _count(self,query,distinct=None):
4646 return 'count %s' % repr(query)
4647
4648 - def _select(self,query,fields,attributes):
4649 return 'select %s where %s' % (repr(fields), repr(query))
4650
4651 - def _delete(self,tablename, query):
4652 return 'delete %s where %s' % (repr(tablename),repr(query))
4653
4654 - def _update(self,tablename,query,fields):
4655 return 'update %s (%s) where %s' % (repr(tablename),
4656 repr(fields),repr(query))
4657
4659 """
4660 remember: no transactions on many NoSQL
4661 """
4662 pass
4663
4665 """
4666 remember: no transactions on many NoSQL
4667 """
4668 pass
4669
4671 """
4672 remember: no transactions on many NoSQL
4673 """
4674 pass
4675
4676
4677
4678 - def OR(self,first,second): raise SyntaxError("Not supported")
4679 - def AND(self,first,second): raise SyntaxError("Not supported")
4680 - def AS(self,first,second): raise SyntaxError("Not supported")
4681 - def ON(self,first,second): raise SyntaxError("Not supported")
4682 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4683 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4684 - def ADD(self,first,second): raise SyntaxError("Not supported")
4685 - def SUB(self,first,second): raise SyntaxError("Not supported")
4686 - def MUL(self,first,second): raise SyntaxError("Not supported")
4687 - def DIV(self,first,second): raise SyntaxError("Not supported")
4688 - def LOWER(self,first): raise SyntaxError("Not supported")
4689 - def UPPER(self,first): raise SyntaxError("Not supported")
4691 - def LENGTH(self, first): raise SyntaxError("Not supported")
4692 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4693 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4694 - def RANDOM(self): raise SyntaxError("Not supported")
4695 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4696 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4697 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4698 - def drop(self,table,mode): raise SyntaxError("Not supported")
4699 - def alias(self,table,alias): raise SyntaxError("Not supported")
4700 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4702 - def prepare(self,key): raise SyntaxError("Not supported")
4705 - def concat_add(self,table): raise SyntaxError("Not supported")
4706 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4708 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4709 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4711 - def lastrowid(self,table): raise SyntaxError("Not supported")
4712 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4713
4714
4715 -class GAEF(object):
4716 - def __init__(self,name,op,value,apply):
4717 self.name=name=='id' and '__key__' or name
4718 self.op=op
4719 self.value=value
4720 self.apply=apply
4722 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4723
4725 """
4726 NDB:
4727
4728 You can enable NDB by using adapter_args:
4729
4730 db = DAL('google:datastore', adapter_args={'ndb_settings':ndb_settings, 'use_ndb':True})
4731
4732 ndb_settings is optional and can be used for per model caching settings.
4733 It must be a dict in this form:
4734 ndb_settings = {<table_name>:{<variable_name>:<variable_value>}}
4735 See: https://developers.google.com/appengine/docs/python/ndb/cache
4736 """
4737
4738 uploads_in_blob = True
4739 types = {}
4740
4742 - def file_open(self, filename, mode='rb', lock=True): pass
4744
4745 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
4746
4747 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4748 credential_decoder=IDENTITY, driver_args={},
4749 adapter_args={}, do_connect=True, after_connection=None):
4750 self.use_ndb = ('use_ndb' in adapter_args) and adapter_args['use_ndb']
4751 if self.use_ndb is True:
4752 self.types.update({
4753 'boolean': ndb.BooleanProperty,
4754 'string': (lambda **kwargs: ndb.StringProperty(**kwargs)),
4755 'text': ndb.TextProperty,
4756 'json': ndb.TextProperty,
4757 'password': ndb.StringProperty,
4758 'blob': ndb.BlobProperty,
4759 'upload': ndb.StringProperty,
4760 'integer': ndb.IntegerProperty,
4761 'bigint': ndb.IntegerProperty,
4762 'float': ndb.FloatProperty,
4763 'double': ndb.FloatProperty,
4764 'decimal': NDBDecimalProperty,
4765 'date': ndb.DateProperty,
4766 'time': ndb.TimeProperty,
4767 'datetime': ndb.DateTimeProperty,
4768 'id': None,
4769 'reference': ndb.IntegerProperty,
4770 'list:string': (lambda **kwargs: ndb.StringProperty(repeated=True,default=None, **kwargs)),
4771 'list:integer': (lambda **kwargs: ndb.IntegerProperty(repeated=True,default=None, **kwargs)),
4772 'list:reference': (lambda **kwargs: ndb.IntegerProperty(repeated=True,default=None, **kwargs)),
4773 })
4774 else:
4775 self.types.update({
4776 'boolean': gae.BooleanProperty,
4777 'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)),
4778 'text': gae.TextProperty,
4779 'json': gae.TextProperty,
4780 'password': gae.StringProperty,
4781 'blob': gae.BlobProperty,
4782 'upload': gae.StringProperty,
4783 'integer': gae.IntegerProperty,
4784 'bigint': gae.IntegerProperty,
4785 'float': gae.FloatProperty,
4786 'double': gae.FloatProperty,
4787 'decimal': GAEDecimalProperty,
4788 'date': gae.DateProperty,
4789 'time': gae.TimeProperty,
4790 'datetime': gae.DateTimeProperty,
4791 'id': None,
4792 'reference': gae.IntegerProperty,
4793 'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)),
4794 'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4795 'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4796 })
4797 self.db = db
4798 self.uri = uri
4799 self.dbengine = 'google:datastore'
4800 self.folder = folder
4801 db['_lastsql'] = ''
4802 self.db_codec = 'UTF-8'
4803 self._after_connection = after_connection
4804 self.pool_size = 0
4805 match = self.REGEX_NAMESPACE.match(uri)
4806 if match:
4807 namespace_manager.set_namespace(match.group('namespace'))
4808 self.keyfunc = (self.use_ndb and ndb.Key) or Key.from_path
4809
4810 self.ndb_settings = None
4811 if 'ndb_settings' in adapter_args:
4812 self.ndb_settings = adapter_args['ndb_settings']
4813
4814 - def parse_id(self, value, field_type):
4816
4817 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4818 myfields = {}
4819 for field in table:
4820 if isinstance(polymodel,Table) and field.name in polymodel.fields():
4821 continue
4822 attr = {}
4823 if isinstance(field.custom_qualifier, dict):
4824
4825 attr = field.custom_qualifier
4826 field_type = field.type
4827 if isinstance(field_type, SQLCustomType):
4828 ftype = self.types[field_type.native or field_type.type](**attr)
4829 elif isinstance(field_type, ((self.use_ndb and ndb.Property) or gae.Property)):
4830 ftype = field_type
4831 elif field_type.startswith('id'):
4832 continue
4833 elif field_type.startswith('decimal'):
4834 precision, scale = field_type[7:].strip('()').split(',')
4835 precision = int(precision)
4836 scale = int(scale)
4837 dec_cls = (self.use_ndb and NDBDecimalProperty) or GAEDecimalProperty
4838 ftype = dec_cls(precision, scale, **attr)
4839 elif field_type.startswith('reference'):
4840 if field.notnull:
4841 attr = dict(required=True)
4842 referenced = field_type[10:].strip()
4843 ftype = self.types[field_type[:9]](referenced, **attr)
4844 elif field_type.startswith('list:reference'):
4845 if field.notnull:
4846 attr['required'] = True
4847 referenced = field_type[15:].strip()
4848 ftype = self.types[field_type[:14]](**attr)
4849 elif field_type.startswith('list:'):
4850 ftype = self.types[field_type](**attr)
4851 elif not field_type in self.types\
4852 or not self.types[field_type]:
4853 raise SyntaxError('Field: unknown field type: %s' % field_type)
4854 else:
4855 ftype = self.types[field_type](**attr)
4856 myfields[field.name] = ftype
4857 if not polymodel:
4858 model_cls = (self.use_ndb and ndb.Model) or gae.Model
4859 table._tableobj = classobj(table._tablename, (model_cls, ), myfields)
4860 if self.use_ndb:
4861
4862 if self.ndb_settings and (table._tablename in self.ndb_settings):
4863 for k, v in self.ndb_settings.iteritems():
4864 setattr(table._tableobj, k, v)
4865 elif polymodel==True:
4866 pm_cls = (self.use_ndb and NDBPolyModel) or PolyModel
4867 table._tableobj = classobj(table._tablename, (pm_cls, ), myfields)
4868 elif isinstance(polymodel,Table):
4869 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
4870 else:
4871 raise SyntaxError("polymodel must be None, True, a table or a tablename")
4872 return None
4873
4874 - def expand(self,expression,field_type=None):
4875 if isinstance(expression,Field):
4876 if expression.type in ('text', 'blob', 'json'):
4877 raise SyntaxError('AppEngine does not index by: %s' % expression.type)
4878 return expression.name
4879 elif isinstance(expression, (Expression, Query)):
4880 if not expression.second is None:
4881 return expression.op(expression.first, expression.second)
4882 elif not expression.first is None:
4883 return expression.op(expression.first)
4884 else:
4885 return expression.op()
4886 elif field_type:
4887 return self.represent(expression,field_type)
4888 elif isinstance(expression,(list,tuple)):
4889 return ','.join([self.represent(item,field_type) for item in expression])
4890 else:
4891 return str(expression)
4892
4893
4894 - def AND(self,first,second):
4900
4901 - def EQ(self,first,second=None):
4902 if isinstance(second, Key):
4903 return [GAEF(first.name,'=',second,lambda a,b:a==b)]
4904 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4905
4906 - def NE(self,first,second=None):
4907 if first.type != 'id':
4908 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
4909 else:
4910 if not second is None:
4911 second = Key.from_path(first._tablename, long(second))
4912 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4913
4914 - def LT(self,first,second=None):
4915 if first.type != 'id':
4916 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
4917 else:
4918 second = Key.from_path(first._tablename, long(second))
4919 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4920
4921 - def LE(self,first,second=None):
4922 if first.type != 'id':
4923 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
4924 else:
4925 second = Key.from_path(first._tablename, long(second))
4926 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4927
4928 - def GT(self,first,second=None):
4929 if first.type != 'id' or second==0 or second == '0':
4930 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
4931 else:
4932 second = Key.from_path(first._tablename, long(second))
4933 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4934
4935 - def GE(self,first,second=None):
4936 if first.type != 'id':
4937 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
4938 else:
4939 second = Key.from_path(first._tablename, long(second))
4940 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4941
4944
4945 - def COMMA(self,first,second):
4947
4948 - def BELONGS(self,first,second=None):
4949 if not isinstance(second,(list, tuple, set)):
4950 raise SyntaxError("Not supported")
4951 if not self.use_ndb:
4952 if isinstance(second,set):
4953 second = list(second)
4954 if first.type == 'id':
4955 second = [Key.from_path(first._tablename, int(i)) for i in second]
4956 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4957
4958 - def CONTAINS(self,first,second,case_sensitive=False):
4963
4964 - def NOT(self,first):
4965 nops = { self.EQ: self.NE,
4966 self.NE: self.EQ,
4967 self.LT: self.GE,
4968 self.GT: self.LE,
4969 self.LE: self.GT,
4970 self.GE: self.LT}
4971 if not isinstance(first,Query):
4972 raise SyntaxError("Not suported")
4973 nop = nops.get(first.op,None)
4974 if not nop:
4975 raise SyntaxError("Not suported %s" % first.op.__name__)
4976 first.op = nop
4977 return self.expand(first)
4978
4981
4982 GAE_FILTER_OPTIONS = {
4983 '=': lambda q, t, p, v: q.filter(getattr(t,p) == v),
4984 '>': lambda q, t, p, v: q.filter(getattr(t,p) > v),
4985 '<': lambda q, t, p, v: q.filter(getattr(t,p) < v),
4986 '<=': lambda q, t, p, v: q.filter(getattr(t,p) <= v),
4987 '>=': lambda q, t, p, v: q.filter(getattr(t,p) >= v),
4988 '!=': lambda q, t, p, v: q.filter(getattr(t,p) != v),
4989 'in': lambda q, t, p, v: q.filter(getattr(t,p).IN(v)),
4990 }
4991
4992 - def filter(self, query, tableobj, prop, op, value):
4994
4995 - def select_raw(self,query,fields=None,attributes=None):
4996 db = self.db
4997 fields = fields or []
4998 attributes = attributes or {}
4999 args_get = attributes.get
5000 new_fields = []
5001 for item in fields:
5002 if isinstance(item,SQLALL):
5003 new_fields += item._table
5004 else:
5005 new_fields.append(item)
5006 fields = new_fields
5007 if query:
5008 tablename = self.get_table(query)
5009 elif fields:
5010 tablename = fields[0].tablename
5011 query = db._adapter.id_query(fields[0].table)
5012 else:
5013 raise SyntaxError("Unable to determine a tablename")
5014
5015 if query:
5016 if use_common_filters(query):
5017 query = self.common_filter(query,[tablename])
5018
5019
5020 tableobj = db[tablename]._tableobj
5021 filters = self.expand(query)
5022
5023 projection = None
5024 if len(db[tablename].fields) == len(fields):
5025
5026 projection = None
5027 elif args_get('projection') == True:
5028 projection = []
5029 for f in fields:
5030 if f.type in ['text', 'blob', 'json']:
5031 raise SyntaxError(
5032 "text and blob field types not allowed in projection queries")
5033 else:
5034 projection.append(f.name)
5035 elif args_get('filterfields') == True:
5036 projection = []
5037 for f in fields:
5038 projection.append(f.name)
5039
5040
5041
5042 query_projection = [
5043 p for p in projection if \
5044 p != db[tablename]._id.name] if projection and \
5045 args_get('projection') == True\
5046 else None
5047
5048 cursor = None
5049 if isinstance(args_get('reusecursor'), str):
5050 cursor = args_get('reusecursor')
5051 if self.use_ndb:
5052 qo = ndb.QueryOptions(projection=query_projection, cursor=cursor)
5053 items = tableobj.query(default_options=qo)
5054 else:
5055 items = gae.Query(tableobj, projection=query_projection,
5056 cursor=cursor)
5057
5058 for filter in filters:
5059 if args_get('projection') == True and \
5060 filter.name in query_projection and \
5061 filter.op in ['=', '<=', '>=']:
5062 raise SyntaxError(
5063 "projection fields cannot have equality filters")
5064 if filter.name=='__key__' and filter.op=='>' and filter.value==0:
5065 continue
5066 elif filter.name=='__key__' and filter.op=='=':
5067 if filter.value==0:
5068 items = []
5069 elif isinstance(filter.value, (self.use_ndb and ndb.Key) or Key):
5070
5071
5072
5073 item = filter.value.get() if self.use_ndb else tableobj.get(filter.value)
5074 items = (item and [item]) or []
5075 else:
5076
5077
5078
5079 item = tableobj.get_by_id(filter.value)
5080 items = (item and [item]) or []
5081 elif isinstance(items,list):
5082 items = [i for i in items if filter.apply(
5083 getattr(item,filter.name),filter.value)]
5084 else:
5085 if filter.name=='__key__' and filter.op != 'in':
5086 if self.use_ndb:
5087 items.order(tableobj._key)
5088 else:
5089 items.order('__key__')
5090 items = self.filter(items, tableobj, filter.name,
5091 filter.op, filter.value) \
5092 if self.use_ndb else \
5093 items.filter('%s %s' % (filter.name,filter.op),
5094 filter.value)
5095
5096 if not isinstance(items,list):
5097 if args_get('left', None):
5098 raise SyntaxError('Set: no left join in appengine')
5099 if args_get('groupby', None):
5100 raise SyntaxError('Set: no groupby in appengine')
5101 orderby = args_get('orderby', False)
5102 if orderby:
5103
5104 if isinstance(orderby, (list, tuple)):
5105 orderby = xorify(orderby)
5106 if isinstance(orderby,Expression):
5107 orderby = self.expand(orderby)
5108 orders = orderby.split(', ')
5109 for order in orders:
5110 if self.use_ndb:
5111
5112 def make_order(o):
5113 s = str(o)
5114 desc = s[0] == '-'
5115 s = (desc and s[1:]) or s
5116 return (desc and -getattr(tableobj, s)) or getattr(tableobj, s)
5117 _order = {'-id':-tableobj._key,'id':tableobj._key}.get(order)
5118 if _order is None:
5119 _order = make_order(order)
5120 items = items.order(_order)
5121 else:
5122 order={'-id':'-__key__','id':'__key__'}.get(order,order)
5123 items = items.order(order)
5124 if args_get('limitby', None):
5125 (lmin, lmax) = attributes['limitby']
5126 (limit, offset) = (lmax - lmin, lmin)
5127 if self.use_ndb:
5128 rows, cursor, more = items.fetch_page(limit,offset=offset)
5129 else:
5130 rows = items.fetch(limit,offset=offset)
5131
5132
5133 if args_get('reusecursor'):
5134 db['_lastcursor'] = cursor if self.use_ndb else items.cursor()
5135 items = rows
5136 return (items, tablename, projection or db[tablename].fields)
5137
5138 - def select(self,query,fields,attributes):
5139 """
5140 This is the GAE version of select. some notes to consider:
5141 - db['_lastsql'] is not set because there is not SQL statement string
5142 for a GAE query
5143 - 'nativeRef' is a magical fieldname used for self references on GAE
5144 - optional attribute 'projection' when set to True will trigger
5145 use of the GAE projection queries. note that there are rules for
5146 what is accepted imposed by GAE: each field must be indexed,
5147 projection queries cannot contain blob or text fields, and you
5148 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
5149 - optional attribute 'filterfields' when set to True web2py will only
5150 parse the explicitly listed fields into the Rows object, even though
5151 all fields are returned in the query. This can be used to reduce
5152 memory usage in cases where true projection queries are not
5153 usable.
5154 - optional attribute 'reusecursor' allows use of cursor with queries
5155 that have the limitby attribute. Set the attribute to True for the
5156 first query, set it to the value of db['_lastcursor'] to continue
5157 a previous query. The user must save the cursor value between
5158 requests, and the filters must be identical. It is up to the user
5159 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
5160 """
5161
5162 (items, tablename, fields) = self.select_raw(query,fields,attributes)
5163
5164 rows = [[(t==self.db[tablename]._id.name and item) or \
5165 (t=='nativeRef' and item) or getattr(item, t) \
5166 for t in fields] for item in items]
5167 colnames = ['%s.%s' % (tablename, t) for t in fields]
5168 processor = attributes.get('processor',self.parse)
5169 return processor(rows,fields,colnames,False)
5170
5171 - def count(self,query,distinct=None,limit=None):
5172 if distinct:
5173 raise RuntimeError("COUNT DISTINCT not supported")
5174 (items, tablename, fields) = self.select_raw(query)
5175
5176 try:
5177 return len(items)
5178 except TypeError:
5179 return items.count(limit=limit)
5180
5181 - def delete(self,tablename, query):
5182 """
5183 This function was changed on 2010-05-04 because according to
5184 http://code.google.com/p/googleappengine/issues/detail?id=3119
5185 GAE no longer supports deleting more than 1000 records.
5186 """
5187
5188 (items, tablename, fields) = self.select_raw(query)
5189
5190 if not isinstance(items,list):
5191
5192
5193 leftitems = items.fetch(1000, keys_only=True)
5194 counter = 0
5195 while len(leftitems):
5196 counter += len(leftitems)
5197 if self.use_ndb:
5198 ndb.delete_multi(leftitems)
5199 else:
5200 gae.delete(leftitems)
5201 leftitems = items.fetch(1000, keys_only=True)
5202 else:
5203 counter = len(items)
5204 if self.use_ndb:
5205 ndb.delete_multi([item.key for item in items])
5206 else:
5207 gae.delete(items)
5208 return counter
5209
5210 - def update(self,tablename,query,update_fields):
5211
5212 (items, tablename, fields) = self.select_raw(query)
5213 counter = 0
5214 for item in items:
5215 for field, value in update_fields:
5216 setattr(item, field.name, self.represent(value,field.type))
5217 item.put()
5218 counter += 1
5219 LOGGER.info(str(counter))
5220 return counter
5221
5222 - def insert(self,table,fields):
5223 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
5224
5225 tmp = table._tableobj(**dfields)
5226 tmp.put()
5227 key = tmp.key if self.use_ndb else tmp.key()
5228 rid = Reference(key.id())
5229 (rid._table, rid._record, rid._gaekey) = (table, None, key)
5230 return rid
5231
5233 parsed_items = []
5234 for item in items:
5235 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
5236 parsed_items.append(table._tableobj(**dfields))
5237 if self.use_ndb:
5238 ndb.put_multi(parsed_items)
5239 else:
5240 gae.put(parsed_items)
5241 return True
5242
5244 return uuid.UUID(uuidv).int
5245
5247 return str(uuid.UUID(int=n))
5248
5250 drivers = ('couchdb',)
5251
5252 uploads_in_blob = True
5253 types = {
5254 'boolean': bool,
5255 'string': str,
5256 'text': str,
5257 'json': str,
5258 'password': str,
5259 'blob': str,
5260 'upload': str,
5261 'integer': long,
5262 'bigint': long,
5263 'float': float,
5264 'double': float,
5265 'date': datetime.date,
5266 'time': datetime.time,
5267 'datetime': datetime.datetime,
5268 'id': long,
5269 'reference': long,
5270 'list:string': list,
5271 'list:integer': list,
5272 'list:reference': list,
5273 }
5274
5276 - def file_open(self, filename, mode='rb', lock=True): pass
5278
5279 - def expand(self,expression,field_type=None):
5280 if isinstance(expression,Field):
5281 if expression.type=='id':
5282 return "%s._id" % expression.tablename
5283 return BaseAdapter.expand(self,expression,field_type)
5284
5285 - def AND(self,first,second):
5287
5288 - def OR(self,first,second):
5290
5291 - def EQ(self,first,second):
5295
5296 - def NE(self,first,second):
5300
5301 - def COMMA(self,first,second):
5303
5305 value = NoSQLAdapter.represent(self, obj, fieldtype)
5306 if fieldtype=='id':
5307 return repr(str(long(value)))
5308 elif fieldtype in ('date','time','datetime','boolean'):
5309 return serializers.json(value)
5310 return repr(not isinstance(value,unicode) and value \
5311 or value and value.encode('utf8'))
5312
5313 - def __init__(self,db,uri='couchdb://127.0.0.1:5984',
5314 pool_size=0,folder=None,db_codec ='UTF-8',
5315 credential_decoder=IDENTITY, driver_args={},
5316 adapter_args={}, do_connect=True, after_connection=None):
5317 self.db = db
5318 self.uri = uri
5319 if do_connect: self.find_driver(adapter_args)
5320 self.dbengine = 'couchdb'
5321 self.folder = folder
5322 db['_lastsql'] = ''
5323 self.db_codec = 'UTF-8'
5324 self._after_connection = after_connection
5325 self.pool_size = pool_size
5326
5327 url='http://'+uri[10:]
5328 def connector(url=url,driver_args=driver_args):
5329 return self.driver.Server(url,**driver_args)
5330 self.reconnect(connector,cursor=False)
5331
5332 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
5333 if migrate:
5334 try:
5335 self.connection.create(table._tablename)
5336 except:
5337 pass
5338
5339 - def insert(self,table,fields):
5346
5347 - def _select(self,query,fields,attributes):
5348 if not isinstance(query,Query):
5349 raise SyntaxError("Not Supported")
5350 for key in set(attributes.keys())-SELECT_ARGS:
5351 raise SyntaxError('invalid select attribute: %s' % key)
5352 new_fields=[]
5353 for item in fields:
5354 if isinstance(item,SQLALL):
5355 new_fields += item._table
5356 else:
5357 new_fields.append(item)
5358 def uid(fd):
5359 return fd=='id' and '_id' or fd
5360 def get(row,fd):
5361 return fd=='id' and long(row['_id']) or row.get(fd,None)
5362 fields = new_fields
5363 tablename = self.get_table(query)
5364 fieldnames = [f.name for f in (fields or self.db[tablename])]
5365 colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
5366 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
5367 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
5368 dict(t=tablename,
5369 query=self.expand(query),
5370 order='%s._id' % tablename,
5371 fields=fields)
5372 return fn, colnames
5373
5374 - def select(self,query,fields,attributes):
5375 if not isinstance(query,Query):
5376 raise SyntaxError("Not Supported")
5377 fn, colnames = self._select(query,fields,attributes)
5378 tablename = colnames[0].split('.')[0]
5379 ctable = self.connection[tablename]
5380 rows = [cols['value'] for cols in ctable.query(fn)]
5381 processor = attributes.get('processor',self.parse)
5382 return processor(rows,fields,colnames,False)
5383
5384 - def delete(self,tablename,query):
5385 if not isinstance(query,Query):
5386 raise SyntaxError("Not Supported")
5387 if query.first.type=='id' and query.op==self.EQ:
5388 id = query.second
5389 tablename = query.first.tablename
5390 assert(tablename == query.first.tablename)
5391 ctable = self.connection[tablename]
5392 try:
5393 del ctable[str(id)]
5394 return 1
5395 except couchdb.http.ResourceNotFound:
5396 return 0
5397 else:
5398 tablename = self.get_table(query)
5399 rows = self.select(query,[self.db[tablename]._id],{})
5400 ctable = self.connection[tablename]
5401 for row in rows:
5402 del ctable[str(row.id)]
5403 return len(rows)
5404
5405 - def update(self,tablename,query,fields):
5406 if not isinstance(query,Query):
5407 raise SyntaxError("Not Supported")
5408 if query.first.type=='id' and query.op==self.EQ:
5409 id = query.second
5410 tablename = query.first.tablename
5411 ctable = self.connection[tablename]
5412 try:
5413 doc = ctable[str(id)]
5414 for key,value in fields:
5415 doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
5416 ctable.save(doc)
5417 return 1
5418 except couchdb.http.ResourceNotFound:
5419 return 0
5420 else:
5421 tablename = self.get_table(query)
5422 rows = self.select(query,[self.db[tablename]._id],{})
5423 ctable = self.connection[tablename]
5424 table = self.db[tablename]
5425 for row in rows:
5426 doc = ctable[str(row.id)]
5427 for key,value in fields:
5428 doc[key.name] = self.represent(value,table[key.name].type)
5429 ctable.save(doc)
5430 return len(rows)
5431
5432 - def count(self,query,distinct=None):
5433 if distinct:
5434 raise RuntimeError("COUNT DISTINCT not supported")
5435 if not isinstance(query,Query):
5436 raise SyntaxError("Not Supported")
5437 tablename = self.get_table(query)
5438 rows = self.select(query,[self.db[tablename]._id],{})
5439 return len(rows)
5440
5442 """
5443 validates that the given text is clean: only contains [0-9a-zA-Z_]
5444 """
5445 if not REGEX_ALPHANUMERIC.match(text):
5446 raise SyntaxError('invalid table or field name: %s' % text)
5447 return text
5448
5450 native_json = True
5451 drivers = ('pymongo',)
5452
5453 uploads_in_blob = False
5454
5455 types = {
5456 'boolean': bool,
5457 'string': str,
5458 'text': str,
5459 'json': str,
5460 'password': str,
5461 'blob': str,
5462 'upload': str,
5463 'integer': long,
5464 'bigint': long,
5465 'float': float,
5466 'double': float,
5467 'date': datetime.date,
5468 'time': datetime.time,
5469 'datetime': datetime.datetime,
5470 'id': long,
5471 'reference': long,
5472 'list:string': list,
5473 'list:integer': list,
5474 'list:reference': list,
5475 }
5476
5477 error_messages = {"javascript_needed": "This must yet be replaced" +
5478 " with javascript in order to work."}
5479
5480 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
5481 pool_size=0, folder=None, db_codec ='UTF-8',
5482 credential_decoder=IDENTITY, driver_args={},
5483 adapter_args={}, do_connect=True, after_connection=None):
5484
5485 self.db = db
5486 self.uri = uri
5487 if do_connect: self.find_driver(adapter_args)
5488 import random
5489 from bson.objectid import ObjectId
5490 from bson.son import SON
5491 import pymongo.uri_parser
5492
5493 m = pymongo.uri_parser.parse_uri(uri)
5494
5495 self.SON = SON
5496 self.ObjectId = ObjectId
5497 self.random = random
5498
5499 self.dbengine = 'mongodb'
5500 self.folder = folder
5501 db['_lastsql'] = ''
5502 self.db_codec = 'UTF-8'
5503 self._after_connection = after_connection
5504 self.pool_size = pool_size
5505
5506
5507 self.minimumreplication = adapter_args.get('minimumreplication',0)
5508
5509
5510
5511
5512 self.safe = adapter_args.get('safe',True)
5513
5514 self.uploads_in_blob = adapter_args.get('uploads_in_blob', False)
5515
5516 if isinstance(m,tuple):
5517 m = {"database" : m[1]}
5518 if m.get('database')==None:
5519 raise SyntaxError("Database is required!")
5520
5521 def connector(uri=self.uri,m=m):
5522
5523 if hasattr(self.driver, "MongoClient"):
5524 Connection = self.driver.MongoClient
5525 else:
5526 Connection = self.driver.Connection
5527 return Connection(uri)[m.get('database')]
5528
5529 self.reconnect(connector,cursor=False)
5530
5532 """ Convert input to a valid Mongodb ObjectId instance
5533
5534 self.object_id("<random>") -> ObjectId (not unique) instance """
5535 if not arg:
5536 arg = 0
5537 if isinstance(arg, basestring):
5538
5539 rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
5540 if arg.isdigit() and (not rawhex):
5541 arg = int(arg)
5542 elif arg == "<random>":
5543 arg = int("0x%sL" % \
5544 "".join([self.random.choice("0123456789abcdef") \
5545 for x in range(24)]), 0)
5546 elif arg.isalnum():
5547 if not arg.startswith("0x"):
5548 arg = "0x%s" % arg
5549 try:
5550 arg = int(arg, 0)
5551 except ValueError, e:
5552 raise ValueError(
5553 "invalid objectid argument string: %s" % e)
5554 else:
5555 raise ValueError("Invalid objectid argument string. " +
5556 "Requires an integer or base 16 value")
5557 elif isinstance(arg, self.ObjectId):
5558 return arg
5559
5560 if not isinstance(arg, (int, long)):
5561 raise TypeError("object_id argument must be of type " +
5562 "ObjectId or an objectid representable integer")
5563 hexvalue = hex(arg)[2:].rstrip('L').zfill(24)
5564 return self.ObjectId(hexvalue)
5565
5567
5568 if isinstance(value, self.ObjectId):
5569 value = long(str(value), 16)
5570 return super(MongoDBAdapter,
5571 self).parse_reference(value, field_type)
5572
5573 - def parse_id(self, value, field_type):
5574 if isinstance(value, self.ObjectId):
5575 value = long(str(value), 16)
5576 return super(MongoDBAdapter,
5577 self).parse_id(value, field_type)
5578
5580
5581 if isinstance(obj, self.ObjectId):
5582 value = obj
5583 else:
5584 value = NoSQLAdapter.represent(self, obj, fieldtype)
5585 if isinstance(obj, (list, tuple)) and \
5586 (not fieldtype == "json" or fieldtype.startswith('list:')):
5587 return value
5588
5589 if fieldtype =='date':
5590 if value == None:
5591 return value
5592
5593 t = datetime.time(0, 0, 0)
5594
5595
5596 return datetime.datetime.combine(value, t)
5597 elif fieldtype == 'time':
5598 if value == None:
5599 return value
5600
5601 d = datetime.date(2000, 1, 1)
5602
5603
5604 return datetime.datetime.combine(d, value)
5605 elif fieldtype == "blob":
5606 if value== None:
5607 return value
5608 from bson import Binary
5609 if not isinstance(value, Binary):
5610 if not isinstance(value, basestring):
5611 return Binary(str(value))
5612 return Binary(value)
5613 return value
5614 elif (isinstance(fieldtype, basestring) and
5615 fieldtype.startswith('list:')):
5616 if fieldtype.startswith('list:reference'):
5617 newval = []
5618 for v in value:
5619 newval.append(self.object_id(v))
5620 return newval
5621 return value
5622 elif ((isinstance(fieldtype, basestring) and
5623 fieldtype.startswith("reference")) or
5624 (isinstance(fieldtype, Table)) or fieldtype=="id"):
5625 value = self.object_id(value)
5626 return value
5627
5628 - def create_table(self, table, migrate=True, fake_migrate=False,
5629 polymodel=None, isCapped=False):
5630 if isCapped:
5631 raise RuntimeError("Not implemented")
5632
5633 - def count(self, query, distinct=None, snapshot=True):
5634 if distinct:
5635 raise RuntimeError("COUNT DISTINCT not supported")
5636 if not isinstance(query,Query):
5637 raise SyntaxError("Not Supported")
5638 tablename = self.get_table(query)
5639 return long(self.select(query,[self.db[tablename]._id], {},
5640 count=True,snapshot=snapshot)['count'])
5641
5642
5643
5644
5645
5646 - def expand(self, expression, field_type=None):
5647 if isinstance(expression, Query):
5648
5649
5650
5651
5652
5653 if isinstance(expression.first,Field) and \
5654 ((expression.first.type == 'id') or \
5655 ("reference" in expression.first.type)):
5656 if expression.first.type == 'id':
5657 expression.first.name = '_id'
5658
5659 if isinstance(expression.second, (tuple, list, set)):
5660 expression.second = [self.object_id(item) for
5661 item in expression.second]
5662 else:
5663 expression.second = self.object_id(expression.second)
5664 result = expression.op(expression.first, expression.second)
5665
5666 if isinstance(expression, Field):
5667 if expression.type=='id':
5668 result = "_id"
5669 else:
5670 result = expression.name
5671 elif isinstance(expression, (Expression, Query)):
5672 if not expression.second is None:
5673 result = expression.op(expression.first, expression.second)
5674 elif not expression.first is None:
5675 result = expression.op(expression.first)
5676 elif not isinstance(expression.op, str):
5677 result = expression.op()
5678 else:
5679 result = expression.op
5680 elif field_type:
5681 result = self.represent(expression,field_type)
5682 elif isinstance(expression,(list,tuple)):
5683 result = ','.join(self.represent(item,field_type) for
5684 item in expression)
5685 else:
5686 result = expression
5687 return result
5688
5689 - def drop(self, table, mode=''):
5692
5693 - def truncate(self, table, mode, safe=None):
5694 if safe == None:
5695 safe=self.safe
5696 ctable = self.connection[table._tablename]
5697 ctable.remove(None, safe=True)
5698
5699 - def _select(self, query, fields, attributes):
5700 if 'for_update' in attributes:
5701 logging.warn('mongodb does not support for_update')
5702 for key in set(attributes.keys())-set(('limitby',
5703 'orderby','for_update')):
5704 if attributes[key]!=None:
5705 logging.warn('select attribute not implemented: %s' % key)
5706
5707 new_fields=[]
5708 mongosort_list = []
5709
5710
5711 orderby = attributes.get('orderby', False)
5712 limitby = attributes.get('limitby', False)
5713
5714 if orderby:
5715 if isinstance(orderby, (list, tuple)):
5716 orderby = xorify(orderby)
5717
5718
5719 for f in self.expand(orderby).split(','):
5720 if f.startswith('-'):
5721 mongosort_list.append((f[1:], -1))
5722 else:
5723 mongosort_list.append((f, 1))
5724 if limitby:
5725 limitby_skip, limitby_limit = limitby[0], int(limitby[1])
5726 else:
5727 limitby_skip = limitby_limit = 0
5728
5729 mongofields_dict = self.SON()
5730 mongoqry_dict = {}
5731 for item in fields:
5732 if isinstance(item, SQLALL):
5733 new_fields += item._table
5734 else:
5735 new_fields.append(item)
5736 fields = new_fields
5737 if isinstance(query,Query):
5738 tablename = self.get_table(query)
5739 elif len(fields) != 0:
5740 tablename = fields[0].tablename
5741 else:
5742 raise SyntaxError("The table name could not be found in " +
5743 "the query nor from the select statement.")
5744 mongoqry_dict = self.expand(query)
5745 fields = fields or self.db[tablename]
5746 for field in fields:
5747 mongofields_dict[field.name] = 1
5748
5749 return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5750 limitby_limit, limitby_skip
5751
5752 - def select(self, query, fields, attributes, count=False,
5753 snapshot=False):
5754
5755 tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5756 limitby_limit, limitby_skip = self._select(query, fields, attributes)
5757 ctable = self.connection[tablename]
5758
5759 if count:
5760 return {'count' : ctable.find(
5761 mongoqry_dict, mongofields_dict,
5762 skip=limitby_skip, limit=limitby_limit,
5763 sort=mongosort_list, snapshot=snapshot).count()}
5764 else:
5765
5766 mongo_list_dicts = ctable.find(mongoqry_dict,
5767 mongofields_dict, skip=limitby_skip,
5768 limit=limitby_limit, sort=mongosort_list,
5769 snapshot=snapshot)
5770 rows = []
5771
5772
5773 colnames = []
5774 newnames = []
5775 for field in fields:
5776 colname = str(field)
5777 colnames.append(colname)
5778 tablename, fieldname = colname.split(".")
5779 if fieldname == "_id":
5780
5781 field.name = "id"
5782 newnames.append(".".join((tablename, field.name)))
5783
5784 for record in mongo_list_dicts:
5785 row=[]
5786 for colname in colnames:
5787 tablename, fieldname = colname.split(".")
5788
5789
5790 if fieldname == "id": fieldname = "_id"
5791 if fieldname in record:
5792 value = record[fieldname]
5793 else:
5794 value = None
5795 row.append(value)
5796 rows.append(row)
5797
5798 processor = attributes.get('processor', self.parse)
5799 result = processor(rows, fields, newnames, False)
5800 return result
5801
5802 - def _insert(self, table, fields):
5810
5811
5812
5813
5814 - def insert(self, table, fields, safe=None):
5821
5822
5823 - def _update(self, tablename, query, fields):
5824 if not isinstance(query, Query):
5825 raise SyntaxError("Not Supported")
5826 filter = None
5827 if query:
5828 filter = self.expand(query)
5829
5830 modify = {'$set': dict((k.name, self.represent(v, k.type)) for
5831 k, v in fields if (not k.name in ("_id", "id")))}
5832 return modify, filter
5833
5834 - def update(self, tablename, query, fields, safe=None):
5835 if safe == None:
5836 safe = self.safe
5837
5838
5839 if not isinstance(query, Query):
5840 raise RuntimeError("Not implemented")
5841 amount = self.count(query, False)
5842 modify, filter = self._update(tablename, query, fields)
5843 try:
5844 result = self.connection[tablename].update(filter,
5845 modify, multi=True, safe=safe)
5846 if safe:
5847 try:
5848
5849 return result["n"]
5850 except (KeyError, AttributeError, TypeError):
5851 return amount
5852 else:
5853 return amount
5854 except Exception, e:
5855
5856 raise RuntimeError("uncaught exception when updating rows: %s" % e)
5857
5858 - def _delete(self, tablename, query):
5859 if not isinstance(query, Query):
5860 raise RuntimeError("query type %s is not supported" % \
5861 type(query))
5862 return self.expand(query)
5863
5864 - def delete(self, tablename, query, safe=None):
5865 if safe is None:
5866 safe = self.safe
5867 amount = 0
5868 amount = self.count(query, False)
5869 filter = self._delete(tablename, query)
5870 self.connection[tablename].remove(filter, safe=safe)
5871 return amount
5872
5874 return [self.insert(table,item) for item in items]
5875
5876
5880
5881
5882 - def NOT(self, first):
5884
5885 - def AND(self,first,second):
5888
5889 - def OR(self,first,second):
5892
5893 - def BELONGS(self, first, second):
5894 if isinstance(second, str):
5895 return {self.expand(first) : {"$in" : [ second[:-1]]} }
5896 elif second==[] or second==() or second==set():
5897 return {1:0}
5898 items = [self.expand(item, first.type) for item in second]
5899 return {self.expand(first) : {"$in" : items} }
5900
5901 - def EQ(self,first,second=None):
5902 result = {}
5903 result[self.expand(first)] = self.expand(second)
5904 return result
5905
5906 - def NE(self, first, second=None):
5907 result = {}
5908 result[self.expand(first)] = {'$ne': self.expand(second)}
5909 return result
5910
5911 - def LT(self,first,second=None):
5912 if second is None:
5913 raise RuntimeError("Cannot compare %s < None" % first)
5914 result = {}
5915 result[self.expand(first)] = {'$lt': self.expand(second)}
5916 return result
5917
5918 - def LE(self,first,second=None):
5919 if second is None:
5920 raise RuntimeError("Cannot compare %s <= None" % first)
5921 result = {}
5922 result[self.expand(first)] = {'$lte': self.expand(second)}
5923 return result
5924
5925 - def GT(self,first,second):
5926 result = {}
5927 result[self.expand(first)] = {'$gt': self.expand(second)}
5928 return result
5929
5930 - def GE(self,first,second=None):
5931 if second is None:
5932 raise RuntimeError("Cannot compare %s >= None" % first)
5933 result = {}
5934 result[self.expand(first)] = {'$gte': self.expand(second)}
5935 return result
5936
5937 - def ADD(self, first, second):
5941
5942 - def SUB(self, first, second):
5946
5947 - def MUL(self, first, second):
5951
5952 - def DIV(self, first, second):
5956
5957 - def MOD(self, first, second):
5961
5962 - def AS(self, first, second):
5963 raise NotImplementedError(self.error_messages["javascript_needed"])
5964 return '%s AS %s' % (self.expand(first), second)
5965
5966
5967
5968
5969 - def ON(self, first, second):
5970 raise NotImplementedError("This is not possible in NoSQL" +
5971 " but can be simulated with a wrapper.")
5972 return '%s ON %s' % (self.expand(first), self.expand(second))
5973
5974
5975
5976
5977 - def COMMA(self, first, second):
5979
5980 - def LIKE(self, first, second):
5984
5985 - def ILIKE(self, first, second):
5986 val = second if isinstance(second,self.ObjectId) else {
5987 '$regex': second.replace('%', ''), '$options': 'i'}
5988 return {self.expand(first): val}
5989
5991
5992 return {self.expand(first): ('/^%s/' % \
5993 self.expand(second, 'string'))}
5994
5996
5997 return {self.expand(first): ('/%s^/' % \
5998 self.expand(second, 'string'))}
5999
6000 - def CONTAINS(self, first, second, case_sensitive=False):
6001
6002
6003
6004 val = second if isinstance(second,self.ObjectId) else \
6005 {'$regex':".*" + re.escape(self.expand(second, 'string')) + ".*"}
6006 return {self.expand(first) : val}
6007
6008 - def LIKE(self, first, second):
6013
6014
6016
6017 import re
6018 return {self.expand(first): {'$regex' : '^' +
6019 re.escape(self.expand(second,
6020 'string'))}}
6021
6022
6024
6025
6026
6027
6028 import re
6029 return {self.expand(first): {'$regex': \
6030 re.escape(self.expand(second, 'string')) + '$'}}
6031
6032
6033 - def CONTAINS(self, first, second, case_sensitive=False):
6034
6035
6036
6037
6038 return {self.expand(first) : {'$regex': \
6039 ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
6040
6043 drivers = ('imaplib',)
6044
6045 """ IMAP server adapter
6046
6047 This class is intended as an interface with
6048 email IMAP servers to perform simple queries in the
6049 web2py DAL query syntax, so email read, search and
6050 other related IMAP mail services (as those implemented
6051 by brands like Google(r), and Yahoo!(r)
6052 can be managed from web2py applications.
6053
6054 The code uses examples by Yuji Tomita on this post:
6055 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
6056 and is based in docs for Python imaplib, python email
6057 and email IETF's (i.e. RFC2060 and RFC3501)
6058
6059 This adapter was tested with a small set of operations with Gmail(r). Other
6060 services requests could raise command syntax and response data issues.
6061
6062 It creates its table and field names "statically",
6063 meaning that the developer should leave the table and field
6064 definitions to the DAL instance by calling the adapter's
6065 .define_tables() method. The tables are defined with the
6066 IMAP server mailbox list information.
6067
6068 .define_tables() returns a dictionary mapping dal tablenames
6069 to the server mailbox names with the following structure:
6070
6071 {<tablename>: str <server mailbox name>}
6072
6073 Here is a list of supported fields:
6074
6075 Field Type Description
6076 ################################################################
6077 uid string
6078 answered boolean Flag
6079 created date
6080 content list:string A list of dict text or html parts
6081 to string
6082 cc string
6083 bcc string
6084 size integer the amount of octets of the message*
6085 deleted boolean Flag
6086 draft boolean Flag
6087 flagged boolean Flag
6088 sender string
6089 recent boolean Flag
6090 seen boolean Flag
6091 subject string
6092 mime string The mime header declaration
6093 email string The complete RFC822 message**
6094 attachments <type list> Each non text part as dict
6095 encoding string The main detected encoding
6096
6097 *At the application side it is measured as the length of the RFC822
6098 message string
6099
6100 WARNING: As row id's are mapped to email sequence numbers,
6101 make sure your imap client web2py app does not delete messages
6102 during select or update actions, to prevent
6103 updating or deleting different messages.
6104 Sequence numbers change whenever the mailbox is updated.
6105 To avoid this sequence numbers issues, it is recommended the use
6106 of uid fields in query references (although the update and delete
6107 in separate actions rule still applies).
6108
6109 # This is the code recommended to start imap support
6110 # at the app's model:
6111
6112 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
6113 imapdb.define_tables()
6114
6115 Here is an (incomplete) list of possible imap commands:
6116
6117 # Count today's unseen messages
6118 # smaller than 6000 octets from the
6119 # inbox mailbox
6120
6121 q = imapdb.INBOX.seen == False
6122 q &= imapdb.INBOX.created == datetime.date.today()
6123 q &= imapdb.INBOX.size < 6000
6124 unread = imapdb(q).count()
6125
6126 # Fetch last query messages
6127 rows = imapdb(q).select()
6128
6129 # it is also possible to filter query select results with limitby and
6130 # sequences of mailbox fields
6131
6132 set.select(<fields sequence>, limitby=(<int>, <int>))
6133
6134 # Mark last query messages as seen
6135 messages = [row.uid for row in rows]
6136 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
6137
6138 # Delete messages in the imap database that have mails from mr. Gumby
6139
6140 deleted = 0
6141 for mailbox in imapdb.tables
6142 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
6143
6144 # It is possible also to mark messages for deletion instead of ereasing them
6145 # directly with set.update(deleted=True)
6146
6147
6148 # This object give access
6149 # to the adapter auto mailbox
6150 # mapped names (which native
6151 # mailbox has what table name)
6152
6153 imapdb.mailboxes <dict> # tablename, server native name pairs
6154
6155 # To retrieve a table native mailbox name use:
6156 imapdb.<table>.mailbox
6157
6158 ### New features v2.4.1:
6159
6160 # Declare mailboxes statically with tablename, name pairs
6161 # This avoids the extra server names retrieval
6162
6163 imapdb.define_tables({"inbox": "INBOX"})
6164
6165 # Selects without content/attachments/email columns will only
6166 # fetch header and flags
6167
6168 imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject)
6169 """
6170
6171 types = {
6172 'string': str,
6173 'text': str,
6174 'date': datetime.date,
6175 'datetime': datetime.datetime,
6176 'id': long,
6177 'boolean': bool,
6178 'integer': int,
6179 'bigint': long,
6180 'blob': str,
6181 'list:string': str,
6182 }
6183
6184 dbengine = 'imap'
6185
6186 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')
6187
6188 - def __init__(self,
6189 db,
6190 uri,
6191 pool_size=0,
6192 folder=None,
6193 db_codec ='UTF-8',
6194 credential_decoder=IDENTITY,
6195 driver_args={},
6196 adapter_args={},
6197 do_connect=True,
6198 after_connection=None):
6199
6200
6201
6202
6203 self.db = db
6204 self.uri = uri
6205 if do_connect: self.find_driver(adapter_args)
6206 self.pool_size=pool_size
6207 self.folder = folder
6208 self.db_codec = db_codec
6209 self._after_connection = after_connection
6210 self.credential_decoder = credential_decoder
6211 self.driver_args = driver_args
6212 self.adapter_args = adapter_args
6213 self.mailbox_size = None
6214 self.static_names = None
6215 self.charset = sys.getfilesystemencoding()
6216
6217 self.imap4 = None
6218 uri = uri.split("://")[1]
6219
6220 """ MESSAGE is an identifier for sequence number"""
6221
6222 self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft',
6223 'flagged': '\\Flagged', 'recent': '\\Recent',
6224 'seen': '\\Seen', 'answered': '\\Answered'}
6225 self.search_fields = {
6226 'id': 'MESSAGE', 'created': 'DATE',
6227 'uid': 'UID', 'sender': 'FROM',
6228 'to': 'TO', 'cc': 'CC',
6229 'bcc': 'BCC', 'content': 'TEXT',
6230 'size': 'SIZE', 'deleted': '\\Deleted',
6231 'draft': '\\Draft', 'flagged': '\\Flagged',
6232 'recent': '\\Recent', 'seen': '\\Seen',
6233 'subject': 'SUBJECT', 'answered': '\\Answered',
6234 'mime': None, 'email': None,
6235 'attachments': None
6236 }
6237
6238 db['_lastsql'] = ''
6239
6240 m = self.REGEX_URI.match(uri)
6241 user = m.group('user')
6242 password = m.group('password')
6243 host = m.group('host')
6244 port = int(m.group('port'))
6245 over_ssl = False
6246 if port==993:
6247 over_ssl = True
6248
6249 driver_args.update(host=host,port=port, password=password, user=user)
6250 def connector(driver_args=driver_args):
6251
6252
6253 if over_ssl:
6254 self.imap4 = self.driver.IMAP4_SSL
6255 else:
6256 self.imap4 = self.driver.IMAP4
6257 connection = self.imap4(driver_args["host"], driver_args["port"])
6258 data = connection.login(driver_args["user"], driver_args["password"])
6259
6260
6261 connection.mailbox_names = None
6262
6263
6264 connection.cursor = lambda : True
6265
6266 return connection
6267
6268 self.db.define_tables = self.define_tables
6269 self.connector = connector
6270 if do_connect: self.reconnect()
6271
6316
6318 last_message = None
6319
6320 if not isinstance(self.connection.mailbox_names, dict):
6321 self.get_mailboxes()
6322 try:
6323 result = self.connection.select(
6324 self.connection.mailbox_names[tablename])
6325 last_message = int(result[1][0])
6326
6327 if last_message == 0:
6328 last_message = 1
6329 except (IndexError, ValueError, TypeError, KeyError):
6330 e = sys.exc_info()[1]
6331 LOGGER.debug("Error retrieving the last mailbox" +
6332 " sequence number. %s" % str(e))
6333 return last_message
6334
6336 if not isinstance(self.connection.mailbox_names, dict):
6337 self.get_mailboxes()
6338
6339
6340 last_message = self.get_last_message(tablename)
6341 result, data = self.connection.uid("search", None, "(ALL)")
6342 uid_list = data[0].strip().split()
6343 if len(uid_list) <= 0:
6344 return None
6345 else:
6346 return (uid_list[0], uid_list[-1])
6347
6349 if add is None:
6350 add = datetime.timedelta()
6351 """ Convert a date object to a string
6352 with d-Mon-Y style for IMAP or the inverse
6353 case
6354
6355 add <timedelta> adds to the date object
6356 """
6357 months = [None, "JAN","FEB","MAR","APR","MAY","JUN",
6358 "JUL", "AUG","SEP","OCT","NOV","DEC"]
6359 if isinstance(date, basestring):
6360
6361 try:
6362 if "," in date:
6363 dayname, datestring = date.split(",")
6364 else:
6365 dayname, datestring = None, date
6366 date_list = datestring.strip().split()
6367 year = int(date_list[2])
6368 month = months.index(date_list[1].upper())
6369 day = int(date_list[0])
6370 hms = map(int, date_list[3].split(":"))
6371 return datetime.datetime(year, month, day,
6372 hms[0], hms[1], hms[2]) + add
6373 except (ValueError, AttributeError, IndexError), e:
6374 LOGGER.error("Could not parse date text: %s. %s" %
6375 (date, e))
6376 return None
6377 elif isinstance(date, (datetime.date, datetime.datetime)):
6378 if imf: date_format = "%a, %d %b %Y %H:%M:%S %z"
6379 else: date_format = "%d-%b-%Y"
6380 return (date + add).strftime(date_format)
6381 else:
6382 return None
6383
6384 @staticmethod
6386 from email.header import decode_header
6387 text, encoding = decode_header(f)[0]
6388 if encoding:
6389 text = text.decode(encoding).encode('utf-8')
6390 return text
6391
6392 - def encode_text(self, text, charset, errors="replace"):
6393 """ convert text for mail to unicode"""
6394 if text is None:
6395 text = ""
6396 else:
6397 if isinstance(text, str):
6398 if charset is None:
6399 text = unicode(text, "utf-8", errors)
6400 else:
6401 text = unicode(text, charset, errors)
6402 else:
6403 raise Exception("Unsupported mail text type %s" % type(text))
6404 return text.encode("utf-8")
6405
6407 charset = message.get_content_charset()
6408 return charset
6409
6411 """ Query the mail database for mailbox names """
6412 if self.static_names:
6413
6414 self.connection.mailbox_names = self.static_names
6415 return self.static_names.keys()
6416
6417 mailboxes_list = self.connection.list()
6418 self.connection.mailbox_names = dict()
6419 mailboxes = list()
6420 x = 0
6421 for item in mailboxes_list[1]:
6422 x = x + 1
6423 item = item.strip()
6424 if not "NOSELECT" in item.upper():
6425 sub_items = item.split("\"")
6426 sub_items = [sub_item for sub_item in sub_items \
6427 if len(sub_item.strip()) > 0]
6428
6429 mailbox = sub_items[-1].strip()
6430
6431
6432 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
6433 mailboxes.append(mailbox_name)
6434 self.connection.mailbox_names[mailbox_name] = mailbox
6435
6436 return mailboxes
6437
6439 nofield = True
6440 tablename = None
6441 attr = query
6442 while nofield:
6443 if hasattr(attr, "first"):
6444 attr = attr.first
6445 if isinstance(attr, Field):
6446 return attr.tablename
6447 elif isinstance(attr, Query):
6448 pass
6449 else:
6450 return None
6451 else:
6452 return None
6453 return tablename
6454
6456 if self.search_fields.get(flag, None) in self.flags.values():
6457 return True
6458 else:
6459 return False
6460
6462 """
6463 Auto create common IMAP fileds
6464
6465 This function creates fields definitions "statically"
6466 meaning that custom fields as in other adapters should
6467 not be supported and definitions handled on a service/mode
6468 basis (local syntax for Gmail(r), Ymail(r)
6469
6470 Returns a dictionary with tablename, server native mailbox name
6471 pairs.
6472 """
6473 if mailbox_names:
6474
6475 self.static_names = mailbox_names
6476 else:
6477 self.static_names = None
6478 if not isinstance(self.connection.mailbox_names, dict):
6479 self.get_mailboxes()
6480
6481 names = self.connection.mailbox_names.keys()
6482
6483 for name in names:
6484 self.db.define_table("%s" % name,
6485 Field("uid", writable=False),
6486 Field("created", "datetime", writable=False),
6487 Field("content", "text", writable=False),
6488 Field("to", writable=False),
6489 Field("cc", writable=False),
6490 Field("bcc", writable=False),
6491 Field("sender", writable=False),
6492 Field("size", "integer", writable=False),
6493 Field("subject", writable=False),
6494 Field("mime", writable=False),
6495 Field("email", "text", writable=False, readable=False),
6496 Field("attachments", "text", writable=False, readable=False),
6497 Field("encoding", writable=False),
6498 Field("answered", "boolean"),
6499 Field("deleted", "boolean"),
6500 Field("draft", "boolean"),
6501 Field("flagged", "boolean"),
6502 Field("recent", "boolean", writable=False),
6503 Field("seen", "boolean")
6504 )
6505
6506
6507
6508 self.db[name].mailbox = \
6509 self.connection.mailbox_names[name]
6510
6511
6512 self.db[name].to.represent = self.db[name].cc.represent = \
6513 self.db[name].bcc.represent = self.db[name].sender.represent = \
6514 self.db[name].subject.represent = self.header_represent
6515
6516
6517 self.db.mailboxes = self.connection.mailbox_names
6518 return self.db.mailboxes
6519
6524
6525 - def _select(self, query, fields, attributes):
6529
6530 - def select(self, query, fields, attributes):
6531 """ Search and Fetch records and return web2py rows
6532 """
6533
6534 if use_common_filters(query):
6535 query = self.common_filter(query, [self.get_query_mailbox(query),])
6536
6537 import email
6538
6539
6540 tablename = None
6541 fetch_results = list()
6542
6543 if isinstance(query, Query):
6544 tablename = self.get_table(query)
6545 mailbox = self.connection.mailbox_names.get(tablename, None)
6546 if mailbox is None:
6547 raise ValueError("Mailbox name not found: %s" % mailbox)
6548 else:
6549
6550 result, selected = self.connection.select(mailbox, True)
6551 if result != "OK":
6552 raise Exception("IMAP error: %s" % selected)
6553 self.mailbox_size = int(selected[0])
6554 search_query = "(%s)" % str(query).strip()
6555 search_result = self.connection.uid("search", None, search_query)
6556
6557 if search_result[0] == "OK":
6558
6559
6560
6561
6562 limitby = attributes.get('limitby', None)
6563 messages_set = search_result[1][0].split()
6564
6565 messages_set.reverse()
6566 if limitby is not None:
6567
6568 messages_set = messages_set[int(limitby[0]):int(limitby[1])]
6569
6570
6571 if any([(field.name in ["content", "size",
6572 "attachments", "email"]) for
6573 field in fields]):
6574 imap_fields = "(RFC822 FLAGS)"
6575 else:
6576 imap_fields = "(RFC822.HEADER FLAGS)"
6577
6578 if len(messages_set) > 0:
6579
6580
6581
6582
6583 for uid in messages_set:
6584
6585 typ, data = self.connection.uid("fetch", uid, imap_fields)
6586 if typ == "OK":
6587 fr = {"message": int(data[0][0].split()[0]),
6588 "uid": long(uid),
6589 "email": email.message_from_string(data[0][1]),
6590 "raw_message": data[0][1]}
6591 fr["multipart"] = fr["email"].is_multipart()
6592
6593 fr["flags"] = self.driver.ParseFlags(data[1])
6594 fetch_results.append(fr)
6595 else:
6596
6597 raise Exception("IMAP error retrieving the body: %s" % data)
6598 else:
6599 raise Exception("IMAP search error: %s" % search_result[1])
6600 elif isinstance(query, (Expression, basestring)):
6601 raise NotImplementedError()
6602 else:
6603 raise TypeError("Unexpected query type")
6604
6605 imapqry_dict = {}
6606 imapfields_dict = {}
6607
6608 if len(fields) == 1 and isinstance(fields[0], SQLALL):
6609 allfields = True
6610 elif len(fields) == 0:
6611 allfields = True
6612 else:
6613 allfields = False
6614 if allfields:
6615 colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
6616 else:
6617 colnames = ["%s.%s" % (tablename, field.name) for field in fields]
6618
6619 for k in colnames:
6620 imapfields_dict[k] = k
6621
6622 imapqry_list = list()
6623 imapqry_array = list()
6624 for fr in fetch_results:
6625 attachments = []
6626 content = []
6627 size = 0
6628 n = int(fr["message"])
6629 item_dict = dict()
6630 message = fr["email"]
6631 uid = fr["uid"]
6632 charset = self.get_charset(message)
6633 flags = fr["flags"]
6634 raw_message = fr["raw_message"]
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645 if "%s.id" % tablename in colnames:
6646 item_dict["%s.id" % tablename] = n
6647 if "%s.created" % tablename in colnames:
6648 item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
6649 if "%s.uid" % tablename in colnames:
6650 item_dict["%s.uid" % tablename] = uid
6651 if "%s.sender" % tablename in colnames:
6652
6653
6654
6655 item_dict["%s.sender" % tablename] = message["From"]
6656 if "%s.to" % tablename in colnames:
6657 item_dict["%s.to" % tablename] = message["To"]
6658 if "%s.cc" % tablename in colnames:
6659 if "Cc" in message.keys():
6660 item_dict["%s.cc" % tablename] = message["Cc"]
6661 else:
6662 item_dict["%s.cc" % tablename] = ""
6663 if "%s.bcc" % tablename in colnames:
6664 if "Bcc" in message.keys():
6665 item_dict["%s.bcc" % tablename] = message["Bcc"]
6666 else:
6667 item_dict["%s.bcc" % tablename] = ""
6668 if "%s.deleted" % tablename in colnames:
6669 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
6670 if "%s.draft" % tablename in colnames:
6671 item_dict["%s.draft" % tablename] = "\\Draft" in flags
6672 if "%s.flagged" % tablename in colnames:
6673 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
6674 if "%s.recent" % tablename in colnames:
6675 item_dict["%s.recent" % tablename] = "\\Recent" in flags
6676 if "%s.seen" % tablename in colnames:
6677 item_dict["%s.seen" % tablename] = "\\Seen" in flags
6678 if "%s.subject" % tablename in colnames:
6679 item_dict["%s.subject" % tablename] = message["Subject"]
6680 if "%s.answered" % tablename in colnames:
6681 item_dict["%s.answered" % tablename] = "\\Answered" in flags
6682 if "%s.mime" % tablename in colnames:
6683 item_dict["%s.mime" % tablename] = message.get_content_type()
6684 if "%s.encoding" % tablename in colnames:
6685 item_dict["%s.encoding" % tablename] = charset
6686
6687
6688
6689
6690
6691
6692 if "%s.email" % tablename in colnames:
6693
6694 item_dict["%s.email" % tablename] = raw_message
6695
6696
6697
6698
6699
6700
6701 for part in message.walk():
6702 maintype = part.get_content_maintype()
6703 if ("%s.attachments" % tablename in colnames) or \
6704 ("%s.content" % tablename in colnames):
6705 payload = part.get_payload(decode=True)
6706 if payload:
6707 filename = part.get_filename()
6708 values = {"mime": part.get_content_type()}
6709 if ((filename or not "text" in maintype) and
6710 ("%s.attachments" % tablename in colnames)):
6711 values.update({"payload": payload,
6712 "filename": filename,
6713 "encoding": part.get_content_charset(),
6714 "disposition": part["Content-Disposition"]})
6715 attachments.append(values)
6716 elif (("text" in maintype) and
6717 ("%s.content" % tablename in colnames)):
6718 values.update({"text": self.encode_text(payload,
6719 self.get_charset(part))})
6720 content.append(values)
6721
6722 if "%s.size" % tablename in colnames:
6723 if part is not None:
6724 size += len(str(part))
6725 item_dict["%s.content" % tablename] = content
6726 item_dict["%s.attachments" % tablename] = attachments
6727 item_dict["%s.size" % tablename] = size
6728 imapqry_list.append(item_dict)
6729
6730
6731
6732 for item_dict in imapqry_list:
6733 imapqry_array_item = list()
6734 for fieldname in colnames:
6735 imapqry_array_item.append(item_dict[fieldname])
6736 imapqry_array.append(imapqry_array_item)
6737
6738
6739 colnames = colnames
6740 processor = attributes.get('processor',self.parse)
6741 return processor(imapqry_array, fields, colnames)
6742
6743 - def _insert(self, table, fields):
6744 def add_payload(message, obj):
6745 payload = Message()
6746 encoding = obj.get("encoding", "utf-8")
6747 if encoding and (encoding.upper() in
6748 ("BASE64", "7BIT", "8BIT", "BINARY")):
6749 payload.add_header("Content-Transfer-Encoding", encoding)
6750 else:
6751 payload.set_charset(encoding)
6752 mime = obj.get("mime", None)
6753 if mime:
6754 payload.set_type(mime)
6755 if "text" in obj:
6756 payload.set_payload(obj["text"])
6757 elif "payload" in obj:
6758 payload.set_payload(obj["payload"])
6759 if "filename" in obj and obj["filename"]:
6760 payload.add_header("Content-Disposition",
6761 "attachment", filename=obj["filename"])
6762 message.attach(payload)
6763
6764 mailbox = table.mailbox
6765 d = dict(((k.name, v) for k, v in fields))
6766 date_time = d.get("created") or datetime.datetime.now()
6767 struct_time = date_time.timetuple()
6768 if len(d) > 0:
6769 message = d.get("email", None)
6770 attachments = d.get("attachments", [])
6771 content = d.get("content", [])
6772 flags = " ".join(["\\%s" % flag.capitalize() for flag in
6773 ("answered", "deleted", "draft", "flagged",
6774 "recent", "seen") if d.get(flag, False)])
6775 if not message:
6776 from email.message import Message
6777 mime = d.get("mime", None)
6778 charset = d.get("encoding", None)
6779 message = Message()
6780 message["from"] = d.get("sender", "")
6781 message["subject"] = d.get("subject", "")
6782 message["date"] = self.convert_date(date_time, imf=True)
6783
6784 if mime:
6785 message.set_type(mime)
6786 if charset:
6787 message.set_charset(charset)
6788 for item in ("to", "cc", "bcc"):
6789 value = d.get(item, "")
6790 if isinstance(value, basestring):
6791 message[item] = value
6792 else:
6793 message[item] = ";".join([i for i in
6794 value])
6795 if (not message.is_multipart() and
6796 (not message.get_content_type().startswith(
6797 "multipart"))):
6798 if isinstance(content, basestring):
6799 message.set_payload(content)
6800 elif len(content) > 0:
6801 message.set_payload(content[0]["text"])
6802 else:
6803 [add_payload(message, c) for c in content]
6804 [add_payload(message, a) for a in attachments]
6805 message = message.as_string()
6806 return (mailbox, flags, struct_time, message)
6807 else:
6808 raise NotImplementedError("IMAP empty insert is not implemented")
6809
6810 - def insert(self, table, fields):
6811 values = self._insert(table, fields)
6812 result, data = self.connection.append(*values)
6813 if result == "OK":
6814 uid = int(re.findall("\d+", str(data))[-1])
6815 return self.db(table.uid==uid).select(table.id).first().id
6816 else:
6817 raise Exception("IMAP message append failed: %s" % data)
6818
6819 - def _update(self, tablename, query, fields, commit=False):
6820
6821 commands = list()
6822 if use_common_filters(query):
6823 query = self.common_filter(query, [tablename,])
6824 mark = []
6825 unmark = []
6826 if query:
6827 for item in fields:
6828 field = item[0]
6829 name = field.name
6830 value = item[1]
6831 if self.is_flag(name):
6832 flag = self.search_fields[name]
6833 if (value is not None) and (flag != "\\Recent"):
6834 if value:
6835 mark.append(flag)
6836 else:
6837 unmark.append(flag)
6838 result, data = self.connection.select(
6839 self.connection.mailbox_names[tablename])
6840 string_query = "(%s)" % query
6841 result, data = self.connection.search(None, string_query)
6842 store_list = [item.strip() for item in data[0].split()
6843 if item.strip().isdigit()]
6844
6845 for number in store_list:
6846 result = None
6847 if len(mark) > 0:
6848 commands.append((number, "+FLAGS", "(%s)" % " ".join(mark)))
6849 if len(unmark) > 0:
6850 commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark)))
6851 return commands
6852
6853 - def update(self, tablename, query, fields):
6854 rowcount = 0
6855 commands = self._update(tablename, query, fields)
6856 for command in commands:
6857 result, data = self.connection.store(*command)
6858 if result == "OK":
6859 rowcount += 1
6860 else:
6861 raise Exception("IMAP storing error: %s" % data)
6862 return rowcount
6863
6864 - def _count(self, query, distinct=None):
6865 raise NotImplementedError()
6866
6867 - def count(self,query,distinct=None):
6879
6880 - def delete(self, tablename, query):
6881 counter = 0
6882 if query:
6883 if use_common_filters(query):
6884 query = self.common_filter(query, [tablename,])
6885 result, data = self.connection.select(self.connection.mailbox_names[tablename])
6886 string_query = "(%s)" % query
6887 result, data = self.connection.search(None, string_query)
6888 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
6889 for number in store_list:
6890 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
6891 if result == "OK":
6892 counter += 1
6893 else:
6894 raise Exception("IMAP store error: %s" % data)
6895 if counter > 0:
6896 result, data = self.connection.expunge()
6897 return counter
6898
6899 - def BELONGS(self, first, second):
6900 result = None
6901 name = self.search_fields[first.name]
6902 if name == "MESSAGE":
6903 values = [str(val) for val in second if str(val).isdigit()]
6904 result = "%s" % ",".join(values).strip()
6905
6906 elif name == "UID":
6907 values = [str(val) for val in second if str(val).isdigit()]
6908 result = "UID %s" % ",".join(values).strip()
6909
6910 else:
6911 raise Exception("Operation not supported")
6912
6913 return result
6914
6915 - def CONTAINS(self, first, second, case_sensitive=False):
6916
6917 result = None
6918 name = self.search_fields[first.name]
6919
6920 if name in ("FROM", "TO", "SUBJECT", "TEXT"):
6921 result = "%s \"%s\"" % (name, self.expand(second))
6922 else:
6923 if first.name in ("cc", "bcc"):
6924 result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
6925 elif first.name == "mime":
6926 result = "HEADER Content-Type \"%s\"" % self.expand(second)
6927 else:
6928 raise Exception("Operation not supported")
6929 return result
6930
6931 - def GT(self, first, second):
6932 result = None
6933 name = self.search_fields[first.name]
6934 if name == "MESSAGE":
6935 last_message = self.get_last_message(first.tablename)
6936 result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
6937 elif name == "UID":
6938
6939
6940
6941 try:
6942 pedestal, threshold = self.get_uid_bounds(first.tablename)
6943 except TypeError:
6944 e = sys.exc_info()[1]
6945 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6946 return ""
6947 try:
6948 lower_limit = int(self.expand(second)) + 1
6949 except (ValueError, TypeError):
6950 e = sys.exc_info()[1]
6951 raise Exception("Operation not supported (non integer UID)")
6952 result = "UID %s:%s" % (lower_limit, threshold)
6953 elif name == "DATE":
6954 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
6955 elif name == "SIZE":
6956 result = "LARGER %s" % self.expand(second)
6957 else:
6958 raise Exception("Operation not supported")
6959 return result
6960
6961 - def GE(self, first, second):
6962 result = None
6963 name = self.search_fields[first.name]
6964 if name == "MESSAGE":
6965 last_message = self.get_last_message(first.tablename)
6966 result = "%s:%s" % (self.expand(second), last_message)
6967 elif name == "UID":
6968
6969
6970
6971 try:
6972 pedestal, threshold = self.get_uid_bounds(first.tablename)
6973 except TypeError:
6974 e = sys.exc_info()[1]
6975 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6976 return ""
6977 lower_limit = self.expand(second)
6978 result = "UID %s:%s" % (lower_limit, threshold)
6979 elif name == "DATE":
6980 result = "SINCE %s" % self.convert_date(second)
6981 else:
6982 raise Exception("Operation not supported")
6983 return result
6984
6985 - def LT(self, first, second):
6986 result = None
6987 name = self.search_fields[first.name]
6988 if name == "MESSAGE":
6989 result = "%s:%s" % (1, int(self.expand(second)) - 1)
6990 elif name == "UID":
6991 try:
6992 pedestal, threshold = self.get_uid_bounds(first.tablename)
6993 except TypeError:
6994 e = sys.exc_info()[1]
6995 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6996 return ""
6997 try:
6998 upper_limit = int(self.expand(second)) - 1
6999 except (ValueError, TypeError):
7000 e = sys.exc_info()[1]
7001 raise Exception("Operation not supported (non integer UID)")
7002 result = "UID %s:%s" % (pedestal, upper_limit)
7003 elif name == "DATE":
7004 result = "BEFORE %s" % self.convert_date(second)
7005 elif name == "SIZE":
7006 result = "SMALLER %s" % self.expand(second)
7007 else:
7008 raise Exception("Operation not supported")
7009 return result
7010
7011 - def LE(self, first, second):
7012 result = None
7013 name = self.search_fields[first.name]
7014 if name == "MESSAGE":
7015 result = "%s:%s" % (1, self.expand(second))
7016 elif name == "UID":
7017 try:
7018 pedestal, threshold = self.get_uid_bounds(first.tablename)
7019 except TypeError:
7020 e = sys.exc_info()[1]
7021 LOGGER.debug("Error requesting uid bounds: %s", str(e))
7022 return ""
7023 upper_limit = int(self.expand(second))
7024 result = "UID %s:%s" % (pedestal, upper_limit)
7025 elif name == "DATE":
7026 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
7027 else:
7028 raise Exception("Operation not supported")
7029 return result
7030
7031 - def NE(self, first, second=None):
7032 if (second is None) and isinstance(first, Field):
7033
7034 if first.type == "id":
7035 return self.GE(first, 1)
7036 result = self.NOT(self.EQ(first, second))
7037 result = result.replace("NOT NOT", "").strip()
7038 return result
7039
7040 - def EQ(self,first,second):
7041 name = self.search_fields[first.name]
7042 result = None
7043 if name is not None:
7044 if name == "MESSAGE":
7045
7046 result = "%s" % self.expand(second)
7047 elif name == "UID":
7048 result = "UID %s" % self.expand(second)
7049 elif name == "DATE":
7050 result = "ON %s" % self.convert_date(second)
7051
7052 elif name in self.flags.values():
7053 if second:
7054 result = "%s" % (name.upper()[1:])
7055 else:
7056 result = "NOT %s" % (name.upper()[1:])
7057 else:
7058 raise Exception("Operation not supported")
7059 else:
7060 raise Exception("Operation not supported")
7061 return result
7062
7063 - def AND(self, first, second):
7066
7067 - def OR(self, first, second):
7070
7071 - def NOT(self, first):
7072 result = "NOT %s" % self.expand(first)
7073 return result
7074
7075
7076
7077
7078
7079 ADAPTERS = {
7080 'sqlite': SQLiteAdapter,
7081 'spatialite': SpatiaLiteAdapter,
7082 'sqlite:memory': SQLiteAdapter,
7083 'spatialite:memory': SpatiaLiteAdapter,
7084 'mysql': MySQLAdapter,
7085 'postgres': PostgreSQLAdapter,
7086 'postgres:psycopg2': PostgreSQLAdapter,
7087 'postgres:pg8000': PostgreSQLAdapter,
7088 'postgres2:psycopg2': NewPostgreSQLAdapter,
7089 'postgres2:pg8000': NewPostgreSQLAdapter,
7090 'oracle': OracleAdapter,
7091 'mssql': MSSQLAdapter,
7092 'mssql2': MSSQL2Adapter,
7093 'mssql3': MSSQL3Adapter,
7094 'mssql4' : MSSQL4Adapter,
7095 'vertica': VerticaAdapter,
7096 'sybase': SybaseAdapter,
7097 'db2': DB2Adapter,
7098 'teradata': TeradataAdapter,
7099 'informix': InformixAdapter,
7100 'informix-se': InformixSEAdapter,
7101 'firebird': FireBirdAdapter,
7102 'firebird_embedded': FireBirdAdapter,
7103 'ingres': IngresAdapter,
7104 'ingresu': IngresUnicodeAdapter,
7105 'sapdb': SAPDBAdapter,
7106 'cubrid': CubridAdapter,
7107 'jdbc:sqlite': JDBCSQLiteAdapter,
7108 'jdbc:sqlite:memory': JDBCSQLiteAdapter,
7109 'jdbc:postgres': JDBCPostgreSQLAdapter,
7110 'gae': GoogleDatastoreAdapter,
7111 'google:datastore': GoogleDatastoreAdapter,
7112 'google:sql': GoogleSQLAdapter,
7113 'couchdb': CouchDBAdapter,
7114 'mongodb': MongoDBAdapter,
7115 'imap': IMAPAdapter
7116 }
7119 """
7120 Field type validation, using web2py's validators mechanism.
7121
7122 makes sure the content of a field is in line with the declared
7123 fieldtype
7124 """
7125 db = field.db
7126 try:
7127 from gluon import validators
7128 except ImportError:
7129 return []
7130 field_type, field_length = field.type, field.length
7131 if isinstance(field_type, SQLCustomType):
7132 if hasattr(field_type, 'validator'):
7133 return field_type.validator
7134 else:
7135 field_type = field_type.type
7136 elif not isinstance(field_type,str):
7137 return []
7138 requires=[]
7139 def ff(r,id):
7140 row=r(id)
7141 if not row:
7142 return id
7143 elif hasattr(r, '_format') and isinstance(r._format,str):
7144 return r._format % row
7145 elif hasattr(r, '_format') and callable(r._format):
7146 return r._format(row)
7147 else:
7148 return id
7149 if field_type in (('string', 'text', 'password')):
7150 requires.append(validators.IS_LENGTH(field_length))
7151 elif field_type == 'json':
7152 requires.append(validators.IS_EMPTY_OR(validators.IS_JSON(native_json=field.db._adapter.native_json)))
7153 elif field_type == 'double' or field_type == 'float':
7154 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
7155 elif field_type == 'integer':
7156 requires.append(validators.IS_INT_IN_RANGE(-2**31, 2**31))
7157 elif field_type == 'bigint':
7158 requires.append(validators.IS_INT_IN_RANGE(-2**63, 2**63))
7159 elif field_type.startswith('decimal'):
7160 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
7161 elif field_type == 'date':
7162 requires.append(validators.IS_DATE())
7163 elif field_type == 'time':
7164 requires.append(validators.IS_TIME())
7165 elif field_type == 'datetime':
7166 requires.append(validators.IS_DATETIME())
7167 elif db and field_type.startswith('reference') and \
7168 field_type.find('.') < 0 and \
7169 field_type[10:] in db.tables:
7170 referenced = db[field_type[10:]]
7171 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
7172 field.represent = field.represent or repr_ref
7173 if hasattr(referenced, '_format') and referenced._format:
7174 requires = validators.IS_IN_DB(db,referenced._id,
7175 referenced._format)
7176 if field.unique:
7177 requires._and = validators.IS_NOT_IN_DB(db,field)
7178 if field.tablename == field_type[10:]:
7179 return validators.IS_EMPTY_OR(requires)
7180 return requires
7181 elif db and field_type.startswith('list:reference') and \
7182 field_type.find('.') < 0 and \
7183 field_type[15:] in db.tables:
7184 referenced = db[field_type[15:]]
7185 def list_ref_repr(ids, row=None, r=referenced, f=ff):
7186 if not ids:
7187 return None
7188 refs = None
7189 db, id = r._db, r._id
7190 if isinstance(db._adapter, GoogleDatastoreAdapter):
7191 def count(values): return db(id.belongs(values)).select(id)
7192 rx = range(0, len(ids), 30)
7193 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
7194 else:
7195 refs = db(id.belongs(ids)).select(id)
7196 return (refs and ', '.join(f(r,x.id) for x in refs) or '')
7197 field.represent = field.represent or list_ref_repr
7198 if hasattr(referenced, '_format') and referenced._format:
7199 requires = validators.IS_IN_DB(db,referenced._id,
7200 referenced._format,multiple=True)
7201 else:
7202 requires = validators.IS_IN_DB(db,referenced._id,
7203 multiple=True)
7204 if field.unique:
7205 requires._and = validators.IS_NOT_IN_DB(db,field)
7206 if not field.notnull:
7207 requires = validators.IS_EMPTY_OR(requires)
7208 return requires
7209 elif field_type.startswith('list:'):
7210 def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
7211 field.represent = field.represent or repr_list
7212 if field.unique:
7213 requires.insert(0,validators.IS_NOT_IN_DB(db,field))
7214 sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
7215 if field.notnull and not field_type[:2] in sff:
7216 requires.insert(0, validators.IS_NOT_EMPTY())
7217 elif not field.notnull and field_type[:2] in sff and requires:
7218 requires[-1] = validators.IS_EMPTY_OR(requires[-1])
7219 return requires
7220
7223 return str(item).replace('|', '||')
7224
7227
7229 if not hasattr(value,'split') and hasattr(value,'read'):
7230 value = value.read()
7231 return [long(x) for x in value.split('|') if x.strip()]
7232
7236
7237
7238 -class Row(object):
7239
7240 """
7241 a dictionary that lets you do d['a'] as well as d.a
7242 this is only used to store a Row
7243 """
7244
7245 __init__ = lambda self,*args,**kwargs: self.__dict__.update(*args,**kwargs)
7246
7248 key=str(k)
7249 _extra = self.__dict__.get('_extra', None)
7250 if _extra is not None:
7251 v = _extra.get(key, DEFAULT)
7252 if v != DEFAULT:
7253 return v
7254 m = REGEX_TABLE_DOT_FIELD.match(key)
7255 if m:
7256 try:
7257 return ogetattr(self, m.group(1))[m.group(2)]
7258 except (KeyError,AttributeError,TypeError):
7259 key = m.group(2)
7260 try:
7261 return ogetattr(self, key)
7262 except (KeyError,AttributeError,TypeError), ae:
7263 try:
7264 self[key] = ogetattr(self,'__get_lazy_reference__')(key)
7265 return self[key]
7266 except:
7267 raise ae
7268
7269 __setitem__ = lambda self, key, value: setattr(self, str(key), value)
7270
7271 __delitem__ = object.__delattr__
7272
7273 __copy__ = lambda self: Row(self)
7274
7275 __call__ = __getitem__
7276
7277
7278 - def get(self, key, default=None):
7279 try:
7280 return self.__getitem__(key)
7281 except(KeyError, AttributeError, TypeError):
7282 return self.__dict__.get(key,default)
7283
7284 has_key = __contains__ = lambda self, key: key in self.__dict__
7285
7286 __nonzero__ = lambda self: len(self.__dict__)>0
7287
7288 update = lambda self, *args, **kwargs: self.__dict__.update(*args, **kwargs)
7289
7290 keys = lambda self: self.__dict__.keys()
7291
7292 items = lambda self: self.__dict__.items()
7293
7294 values = lambda self: self.__dict__.values()
7295
7296 __iter__ = lambda self: self.__dict__.__iter__()
7297
7298 iteritems = lambda self: self.__dict__.iteritems()
7299
7300 __str__ = __repr__ = lambda self: '<Row %s>' % self.as_dict()
7301
7302 __int__ = lambda self: object.__getattribute__(self,'id')
7303
7304 __long__ = lambda self: long(object.__getattribute__(self,'id'))
7305
7306 __getattr__ = __getitem__
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7318 try:
7319 return self.as_dict() == other.as_dict()
7320 except AttributeError:
7321 return False
7322
7324 return not (self == other)
7325
7327 return Row(dict(self))
7328
7329 - def as_dict(self, datetime_to_str=False, custom_types=None):
7330 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict]
7331 if isinstance(custom_types,(list,tuple,set)):
7332 SERIALIZABLE_TYPES += custom_types
7333 elif custom_types:
7334 SERIALIZABLE_TYPES.append(custom_types)
7335 d = dict(self)
7336 for k in copy.copy(d.keys()):
7337 v=d[k]
7338 if d[k] is None:
7339 continue
7340 elif isinstance(v,Row):
7341 d[k]=v.as_dict()
7342 elif isinstance(v,Reference):
7343 d[k]=long(v)
7344 elif isinstance(v,decimal.Decimal):
7345 d[k]=float(v)
7346 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
7347 if datetime_to_str:
7348 d[k] = v.isoformat().replace('T',' ')[:19]
7349 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)):
7350 del d[k]
7351 return d
7352
7353 - def as_xml(self, row_name="row", colnames=None, indent=' '):
7354 def f(row,field,indent=' '):
7355 if isinstance(row,Row):
7356 spc = indent+' \n'
7357 items = [f(row[x],x,indent+' ') for x in row]
7358 return '%s<%s>\n%s\n%s</%s>' % (
7359 indent,
7360 field,
7361 spc.join(item for item in items if item),
7362 indent,
7363 field)
7364 elif not callable(row):
7365 if REGEX_ALPHANUMERIC.match(field):
7366 return '%s<%s>%s</%s>' % (indent,field,row,field)
7367 else:
7368 return '%s<extra name="%s">%s</extra>' % \
7369 (indent,field,row)
7370 else:
7371 return None
7372 return f(self, row_name, indent=indent)
7373
7374 - def as_json(self, mode="object", default=None, colnames=None,
7375 serialize=True, **kwargs):
7376 """
7377 serializes the row to a JSON object
7378 kwargs are passed to .as_dict method
7379 only "object" mode supported
7380
7381 serialize = False used by Rows.as_json
7382 TODO: return array mode with query column order
7383
7384 mode and colnames are not implemented
7385 """
7386
7387 item = self.as_dict(**kwargs)
7388 if serialize:
7389 if have_serializers:
7390 return serializers.json(item,
7391 default=default or
7392 serializers.custom_json)
7393 elif simplejson:
7394 return simplejson.dumps(item)
7395 else:
7396 raise RuntimeError("missing simplejson")
7397 else:
7398 return item
7399
7409
7411 if not isinstance(fields,(list,tuple)):
7412 fields = [fields]
7413 new_fields = []
7414 for field in fields:
7415 if isinstance(field,Field):
7416 new_fields.append(field)
7417 elif isinstance(field,Table):
7418 for ofield in field:
7419 new_fields.append(ofield)
7420 else:
7421 raise RuntimeError("fields must be a list of fields")
7422 fields = new_fields
7423 field_map = {}
7424 for field in fields:
7425 n = field.name.lower()
7426 if not n in field_map:
7427 field_map[n] = field
7428 n = str(field).lower()
7429 if not n in field_map:
7430 field_map[n] = field
7431 constants = {}
7432 i = 0
7433 while True:
7434 m = REGEX_CONST_STRING.search(text)
7435 if not m: break
7436 text = text[:m.start()]+('#%i' % i)+text[m.end():]
7437 constants[str(i)] = m.group()[1:-1]
7438 i+=1
7439 text = re.sub('\s+',' ',text).lower()
7440 for a,b in [('&','and'),
7441 ('|','or'),
7442 ('~','not'),
7443 ('==','='),
7444 ('<','<'),
7445 ('>','>'),
7446 ('<=','<='),
7447 ('>=','>='),
7448 ('<>','!='),
7449 ('=<','<='),
7450 ('=>','>='),
7451 ('=','='),
7452 (' less or equal than ','<='),
7453 (' greater or equal than ','>='),
7454 (' equal or less than ','<='),
7455 (' equal or greater than ','>='),
7456 (' less or equal ','<='),
7457 (' greater or equal ','>='),
7458 (' equal or less ','<='),
7459 (' equal or greater ','>='),
7460 (' not equal to ','!='),
7461 (' not equal ','!='),
7462 (' equal to ','='),
7463 (' equal ','='),
7464 (' equals ','='),
7465 (' less than ','<'),
7466 (' greater than ','>'),
7467 (' starts with ','startswith'),
7468 (' ends with ','endswith'),
7469 (' not in ' , 'notbelongs'),
7470 (' in ' , 'belongs'),
7471 (' is ','=')]:
7472 if a[0]==' ':
7473 text = text.replace(' is'+a,' %s ' % b)
7474 text = text.replace(a,' %s ' % b)
7475 text = re.sub('\s+',' ',text).lower()
7476 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
7477 query = field = neg = op = logic = None
7478 for item in text.split():
7479 if field is None:
7480 if item == 'not':
7481 neg = True
7482 elif not neg and not logic and item in ('and','or'):
7483 logic = item
7484 elif item in field_map:
7485 field = field_map[item]
7486 else:
7487 raise RuntimeError("Invalid syntax")
7488 elif not field is None and op is None:
7489 op = item
7490 elif not op is None:
7491 if item.startswith('#'):
7492 if not item[1:] in constants:
7493 raise RuntimeError("Invalid syntax")
7494 value = constants[item[1:]]
7495 else:
7496 value = item
7497 if field.type in ('text', 'string', 'json'):
7498 if op == '=': op = 'like'
7499 if op == '=': new_query = field==value
7500 elif op == '<': new_query = field<value
7501 elif op == '>': new_query = field>value
7502 elif op == '<=': new_query = field<=value
7503 elif op == '>=': new_query = field>=value
7504 elif op == '!=': new_query = field!=value
7505 elif op == 'belongs': new_query = field.belongs(value.split(','))
7506 elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
7507 elif field.type in ('text', 'string', 'json'):
7508 if op == 'contains': new_query = field.contains(value)
7509 elif op == 'like': new_query = field.like(value)
7510 elif op == 'startswith': new_query = field.startswith(value)
7511 elif op == 'endswith': new_query = field.endswith(value)
7512 else: raise RuntimeError("Invalid operation")
7513 elif field._db._adapter.dbengine=='google:datastore' and \
7514 field.type in ('list:integer', 'list:string', 'list:reference'):
7515 if op == 'contains': new_query = field.contains(value)
7516 else: raise RuntimeError("Invalid operation")
7517 else: raise RuntimeError("Invalid operation")
7518 if neg: new_query = ~new_query
7519 if query is None:
7520 query = new_query
7521 elif logic == 'and':
7522 query &= new_query
7523 elif logic == 'or':
7524 query |= new_query
7525 field = op = neg = logic = None
7526 return query
7527
7529
7530 """
7531 an instance of this class represents a database connection
7532
7533 Example::
7534
7535 db = DAL('sqlite://test.db')
7536
7537 or
7538
7539 db = DAL(**{"uri": ..., "tables": [...]...}) # experimental
7540
7541 db.define_table('tablename', Field('fieldname1'),
7542 Field('fieldname2'))
7543 """
7544
7545 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
7572
7573 @staticmethod
7575 """
7576 # ## this allows gluon to set a folder for this thread
7577 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
7578 """
7579 BaseAdapter.set_folder(folder)
7580
7581 @staticmethod
7583 """
7584 Returns a dictionary with uri as key with timings and defined tables
7585 {'sqlite://storage.sqlite': {
7586 'dbstats': [(select auth_user.email from auth_user, 0.02009)],
7587 'dbtables': {
7588 'defined': ['auth_cas', 'auth_event', 'auth_group',
7589 'auth_membership', 'auth_permission', 'auth_user'],
7590 'lazy': '[]'
7591 }
7592 }
7593 }
7594 """
7595 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
7596 infos = {}
7597 for db_uid, db_group in dbs:
7598 for db in db_group:
7599 if not db._uri:
7600 continue
7601 k = hide_password(db._adapter.uri)
7602 infos[k] = dict(
7603 dbstats = [(row[0], row[1]) for row in db._timings],
7604 dbtables = {'defined': sorted(
7605 list(set(db.tables)-set(db._LAZY_TABLES.keys()))),
7606 'lazy': sorted(db._LAZY_TABLES.keys())})
7607 return infos
7608
7609 @staticmethod
7622
7623 @staticmethod
7645
7646 - def __init__(self, uri=DEFAULT_URI,
7647 pool_size=0, folder=None,
7648 db_codec='UTF-8', check_reserved=None,
7649 migrate=True, fake_migrate=False,
7650 migrate_enabled=True, fake_migrate_all=False,
7651 decode_credentials=False, driver_args=None,
7652 adapter_args=None, attempts=5, auto_import=False,
7653 bigint_id=False, debug=False, lazy_tables=False,
7654 db_uid=None, do_connect=True,
7655 after_connection=None, tables=None):
7656 """
7657 Creates a new Database Abstraction Layer instance.
7658
7659 Keyword arguments:
7660
7661 :uri: string that contains information for connecting to a database.
7662 (default: 'sqlite://dummy.db')
7663
7664 experimental: you can specify a dictionary as uri
7665 parameter i.e. with
7666 db = DAL({"uri": "sqlite://storage.sqlite",
7667 "tables": {...}, ...})
7668
7669 for an example of dict input you can check the output
7670 of the scaffolding db model with
7671
7672 db.as_dict()
7673
7674 Note that for compatibility with Python older than
7675 version 2.6.5 you should cast your dict input keys
7676 to str due to a syntax limitation on kwarg names.
7677 for proper DAL dictionary input you can use one of:
7678
7679 obj = serializers.cast_keys(dict, [encoding="utf-8"])
7680
7681 or else (for parsing json input)
7682
7683 obj = serializers.loads_json(data, unicode_keys=False)
7684
7685 :pool_size: How many open connections to make to the database object.
7686 :folder: where .table files will be created.
7687 automatically set within web2py
7688 use an explicit path when using DAL outside web2py
7689 :db_codec: string encoding of the database (default: 'UTF-8')
7690 :check_reserved: list of adapters to check tablenames and column names
7691 against sql/nosql reserved keywords. (Default None)
7692
7693 * 'common' List of sql keywords that are common to all database types
7694 such as "SELECT, INSERT". (recommended)
7695 * 'all' Checks against all known SQL keywords. (not recommended)
7696 <adaptername> Checks against the specific adapters list of keywords
7697 (recommended)
7698 * '<adaptername>_nonreserved' Checks against the specific adapters
7699 list of nonreserved keywords. (if available)
7700 :migrate (defaults to True) sets default migrate behavior for all tables
7701 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
7702 :migrate_enabled (defaults to True). If set to False disables ALL migrations
7703 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
7704 :attempts (defaults to 5). Number of times to attempt connecting
7705 :auto_import (defaults to False). If set, import automatically table definitions from the
7706 databases folder
7707 :bigint_id (defaults to False): If set, turn on bigint instead of int for id fields
7708 :lazy_tables (defaults to False): delay table definition until table access
7709 :after_connection (defaults to None): a callable that will be execute after the connection
7710 """
7711 if uri == '<zombie>' and db_uid is not None: return
7712 if not decode_credentials:
7713 credential_decoder = lambda cred: cred
7714 else:
7715 credential_decoder = lambda cred: urllib.unquote(cred)
7716 self._folder = folder
7717 if folder:
7718 self.set_folder(folder)
7719 self._uri = uri
7720 self._pool_size = pool_size
7721 self._db_codec = db_codec
7722 self._lastsql = ''
7723 self._timings = []
7724 self._pending_references = {}
7725 self._request_tenant = 'request_tenant'
7726 self._common_fields = []
7727 self._referee_name = '%(table)s'
7728 self._bigint_id = bigint_id
7729 self._debug = debug
7730 self._migrated = []
7731 self._LAZY_TABLES = {}
7732 self._lazy_tables = lazy_tables
7733 self._tables = SQLCallableList()
7734 self._driver_args = driver_args
7735 self._adapter_args = adapter_args
7736 self._check_reserved = check_reserved
7737 self._decode_credentials = decode_credentials
7738 self._attempts = attempts
7739 self._do_connect = do_connect
7740
7741 if not str(attempts).isdigit() or attempts < 0:
7742 attempts = 5
7743 if uri:
7744 uris = isinstance(uri,(list,tuple)) and uri or [uri]
7745 error = ''
7746 connected = False
7747 for k in range(attempts):
7748 for uri in uris:
7749 try:
7750 if is_jdbc and not uri.startswith('jdbc:'):
7751 uri = 'jdbc:'+uri
7752 self._dbname = REGEX_DBNAME.match(uri).group()
7753 if not self._dbname in ADAPTERS:
7754 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname)
7755
7756
7757 kwargs = dict(db=self,uri=uri,
7758 pool_size=pool_size,
7759 folder=folder,
7760 db_codec=db_codec,
7761 credential_decoder=credential_decoder,
7762 driver_args=driver_args or {},
7763 adapter_args=adapter_args or {},
7764 do_connect=do_connect,
7765 after_connection=after_connection)
7766 self._adapter = ADAPTERS[self._dbname](**kwargs)
7767 types = ADAPTERS[self._dbname].types
7768
7769 self._adapter.types = copy.copy(types)
7770 self._adapter.build_parsemap()
7771 if bigint_id:
7772 if 'big-id' in types and 'reference' in types:
7773 self._adapter.types['id'] = types['big-id']
7774 self._adapter.types['reference'] = types['big-reference']
7775 connected = True
7776 break
7777 except SyntaxError:
7778 raise
7779 except Exception:
7780 tb = traceback.format_exc()
7781 LOGGER.debug('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
7782 if connected:
7783 break
7784 else:
7785 time.sleep(1)
7786 if not connected:
7787 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb))
7788 else:
7789 self._adapter = BaseAdapter(db=self,pool_size=0,
7790 uri='None',folder=folder,
7791 db_codec=db_codec, after_connection=after_connection)
7792 migrate = fake_migrate = False
7793 adapter = self._adapter
7794 self._uri_hash = hashlib_md5(adapter.uri).hexdigest()
7795 self.check_reserved = check_reserved
7796 if self.check_reserved:
7797 from reserved_sql_keywords import ADAPTERS as RSK
7798 self.RSK = RSK
7799 self._migrate = migrate
7800 self._fake_migrate = fake_migrate
7801 self._migrate_enabled = migrate_enabled
7802 self._fake_migrate_all = fake_migrate_all
7803 if auto_import or tables:
7804 self.import_table_definitions(adapter.folder,
7805 tables=tables)
7806
7807 @property
7810
7813 pattern = pjoin(path,self._uri_hash+'_*.table')
7814 if tables:
7815 for table in tables:
7816 self.define_table(**table)
7817 else:
7818 for filename in glob.glob(pattern):
7819 tfile = self._adapter.file_open(filename, 'r')
7820 try:
7821 sql_fields = pickle.load(tfile)
7822 name = filename[len(pattern)-7:-6]
7823 mf = [(value['sortable'],
7824 Field(key,
7825 type=value['type'],
7826 length=value.get('length',None),
7827 notnull=value.get('notnull',False),
7828 unique=value.get('unique',False))) \
7829 for key, value in sql_fields.iteritems()]
7830 mf.sort(lambda a,b: cmp(a[0],b[0]))
7831 self.define_table(name,*[item[1] for item in mf],
7832 **dict(migrate=migrate,
7833 fake_migrate=fake_migrate))
7834 finally:
7835 self._adapter.file_close(tfile)
7836
7838 """
7839 Validates ``name`` against SQL keywords
7840 Uses self.check_reserve which is a list of
7841 operators to use.
7842 self.check_reserved
7843 ['common', 'postgres', 'mysql']
7844 self.check_reserved
7845 ['all']
7846 """
7847 for backend in self.check_reserved:
7848 if name.upper() in self.RSK[backend]:
7849 raise SyntaxError(
7850 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper()))
7851
7852 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
7853 """
7854 EXAMPLE:
7855
7856 db.define_table('person',Field('name'),Field('info'))
7857 db.define_table('pet',Field('ownedby',db.person),Field('name'),Field('info'))
7858
7859 @request.restful()
7860 def index():
7861 def GET(*args,**vars):
7862 patterns = [
7863 "/friends[person]",
7864 "/{person.name}/:field",
7865 "/{person.name}/pets[pet.ownedby]",
7866 "/{person.name}/pets[pet.ownedby]/{pet.name}",
7867 "/{person.name}/pets[pet.ownedby]/{pet.name}/:field",
7868 ("/dogs[pet]", db.pet.info=='dog'),
7869 ("/dogs[pet]/{pet.name.startswith}", db.pet.info=='dog'),
7870 ]
7871 parser = db.parse_as_rest(patterns,args,vars)
7872 if parser.status == 200:
7873 return dict(content=parser.response)
7874 else:
7875 raise HTTP(parser.status,parser.error)
7876
7877 def POST(table_name,**vars):
7878 if table_name == 'person':
7879 return db.person.validate_and_insert(**vars)
7880 elif table_name == 'pet':
7881 return db.pet.validate_and_insert(**vars)
7882 else:
7883 raise HTTP(400)
7884 return locals()
7885 """
7886
7887 db = self
7888 re1 = REGEX_SEARCH_PATTERN
7889 re2 = REGEX_SQUARE_BRACKETS
7890
7891 def auto_table(table,base='',depth=0):
7892 patterns = []
7893 for field in db[table].fields:
7894 if base:
7895 tag = '%s/%s' % (base,field.replace('_','-'))
7896 else:
7897 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
7898 f = db[table][field]
7899 if not f.readable: continue
7900 if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
7901 tag += '/{%s.%s}' % (table,field)
7902 patterns.append(tag)
7903 patterns.append(tag+'/:field')
7904 elif f.type.startswith('boolean'):
7905 tag += '/{%s.%s}' % (table,field)
7906 patterns.append(tag)
7907 patterns.append(tag+'/:field')
7908 elif f.type in ('float','double','integer','bigint'):
7909 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
7910 patterns.append(tag)
7911 patterns.append(tag+'/:field')
7912 elif f.type.startswith('list:'):
7913 tag += '/{%s.%s.contains}' % (table,field)
7914 patterns.append(tag)
7915 patterns.append(tag+'/:field')
7916 elif f.type in ('date','datetime'):
7917 tag+= '/{%s.%s.year}' % (table,field)
7918 patterns.append(tag)
7919 patterns.append(tag+'/:field')
7920 tag+='/{%s.%s.month}' % (table,field)
7921 patterns.append(tag)
7922 patterns.append(tag+'/:field')
7923 tag+='/{%s.%s.day}' % (table,field)
7924 patterns.append(tag)
7925 patterns.append(tag+'/:field')
7926 if f.type in ('datetime','time'):
7927 tag+= '/{%s.%s.hour}' % (table,field)
7928 patterns.append(tag)
7929 patterns.append(tag+'/:field')
7930 tag+='/{%s.%s.minute}' % (table,field)
7931 patterns.append(tag)
7932 patterns.append(tag+'/:field')
7933 tag+='/{%s.%s.second}' % (table,field)
7934 patterns.append(tag)
7935 patterns.append(tag+'/:field')
7936 if depth>0:
7937 for f in db[table]._referenced_by:
7938 tag+='/%s[%s.%s]' % (table,f.tablename,f.name)
7939 patterns.append(tag)
7940 patterns += auto_table(table,base=tag,depth=depth-1)
7941 return patterns
7942
7943 if patterns == 'auto':
7944 patterns=[]
7945 for table in db.tables:
7946 if not table.startswith('auth_'):
7947 patterns.append('/%s[%s]' % (table,table))
7948 patterns += auto_table(table,base='',depth=1)
7949 else:
7950 i = 0
7951 while i<len(patterns):
7952 pattern = patterns[i]
7953 if not isinstance(pattern,str):
7954 pattern = pattern[0]
7955 tokens = pattern.split('/')
7956 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
7957 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
7958 '/'.join(tokens[:-1]))
7959 patterns = patterns[:i]+new_patterns+patterns[i+1:]
7960 i += len(new_patterns)
7961 else:
7962 i += 1
7963 if '/'.join(args) == 'patterns':
7964 return Row({'status':200,'pattern':'list',
7965 'error':None,'response':patterns})
7966 for pattern in patterns:
7967 basequery, exposedfields = None, []
7968 if isinstance(pattern,tuple):
7969 if len(pattern)==2:
7970 pattern, basequery = pattern
7971 elif len(pattern)>2:
7972 pattern, basequery, exposedfields = pattern[0:3]
7973 otable=table=None
7974 if not isinstance(queries,dict):
7975 dbset=db(queries)
7976 if basequery is not None:
7977 dbset = dbset(basequery)
7978 i=0
7979 tags = pattern[1:].split('/')
7980 if len(tags)!=len(args):
7981 continue
7982 for tag in tags:
7983 if re1.match(tag):
7984
7985 tokens = tag[1:-1].split('.')
7986 table, field = tokens[0], tokens[1]
7987 if not otable or table == otable:
7988 if len(tokens)==2 or tokens[2]=='eq':
7989 query = db[table][field]==args[i]
7990 elif tokens[2]=='ne':
7991 query = db[table][field]!=args[i]
7992 elif tokens[2]=='lt':
7993 query = db[table][field]<args[i]
7994 elif tokens[2]=='gt':
7995 query = db[table][field]>args[i]
7996 elif tokens[2]=='ge':
7997 query = db[table][field]>=args[i]
7998 elif tokens[2]=='le':
7999 query = db[table][field]<=args[i]
8000 elif tokens[2]=='year':
8001 query = db[table][field].year()==args[i]
8002 elif tokens[2]=='month':
8003 query = db[table][field].month()==args[i]
8004 elif tokens[2]=='day':
8005 query = db[table][field].day()==args[i]
8006 elif tokens[2]=='hour':
8007 query = db[table][field].hour()==args[i]
8008 elif tokens[2]=='minute':
8009 query = db[table][field].minutes()==args[i]
8010 elif tokens[2]=='second':
8011 query = db[table][field].seconds()==args[i]
8012 elif tokens[2]=='startswith':
8013 query = db[table][field].startswith(args[i])
8014 elif tokens[2]=='contains':
8015 query = db[table][field].contains(args[i])
8016 else:
8017 raise RuntimeError("invalid pattern: %s" % pattern)
8018 if len(tokens)==4 and tokens[3]=='not':
8019 query = ~query
8020 elif len(tokens)>=4:
8021 raise RuntimeError("invalid pattern: %s" % pattern)
8022 if not otable and isinstance(queries,dict):
8023 dbset = db(queries[table])
8024 if basequery is not None:
8025 dbset = dbset(basequery)
8026 dbset=dbset(query)
8027 else:
8028 raise RuntimeError("missing relation in pattern: %s" % pattern)
8029 elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
8030 ref = tag[tag.find('[')+1:-1]
8031 if '.' in ref and otable:
8032 table,field = ref.split('.')
8033 selfld = '_id'
8034 if db[table][field].type.startswith('reference '):
8035 refs = [ x.name for x in db[otable] if x.type == db[table][field].type ]
8036 else:
8037 refs = [ x.name for x in db[table]._referenced_by if x.tablename==otable ]
8038 if refs:
8039 selfld = refs[0]
8040 if nested_select:
8041 try:
8042 dbset=db(db[table][field].belongs(dbset._select(db[otable][selfld])))
8043 except ValueError:
8044 return Row({'status':400,'pattern':pattern,
8045 'error':'invalid path','response':None})
8046 else:
8047 items = [item.id for item in dbset.select(db[otable][selfld])]
8048 dbset=db(db[table][field].belongs(items))
8049 else:
8050 table = ref
8051 if not otable and isinstance(queries,dict):
8052 dbset = db(queries[table])
8053 dbset=dbset(db[table])
8054 elif tag==':field' and table:
8055
8056 field = args[i]
8057 if not field in db[table]: break
8058
8059 if not db[table][field].readable:
8060 return Row({'status':418,'pattern':pattern,
8061 'error':'I\'m a teapot','response':None})
8062 try:
8063 distinct = vars.get('distinct', False) == 'True'
8064 offset = long(vars.get('offset',None) or 0)
8065 limits = (offset,long(vars.get('limit',None) or 1000)+offset)
8066 except ValueError:
8067 return Row({'status':400,'error':'invalid limits','response':None})
8068 items = dbset.select(db[table][field], distinct=distinct, limitby=limits)
8069 if items:
8070 return Row({'status':200,'response':items,
8071 'pattern':pattern})
8072 else:
8073 return Row({'status':404,'pattern':pattern,
8074 'error':'no record found','response':None})
8075 elif tag != args[i]:
8076 break
8077 otable = table
8078 i += 1
8079 if i==len(tags) and table:
8080 ofields = vars.get('order',db[table]._id.name).split('|')
8081 try:
8082 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
8083 except (KeyError, AttributeError):
8084 return Row({'status':400,'error':'invalid orderby','response':None})
8085 if exposedfields:
8086 fields = [field for field in db[table] if str(field).split('.')[-1] in exposedfields and field.readable]
8087 else:
8088 fields = [field for field in db[table] if field.readable]
8089 count = dbset.count()
8090 try:
8091 offset = long(vars.get('offset',None) or 0)
8092 limits = (offset,long(vars.get('limit',None) or 1000)+offset)
8093 except ValueError:
8094 return Row({'status':400,'error':'invalid limits','response':None})
8095 if count > limits[1]-limits[0]:
8096 return Row({'status':400,'error':'too many records','response':None})
8097 try:
8098 response = dbset.select(limitby=limits,orderby=orderby,*fields)
8099 except ValueError:
8100 return Row({'status':400,'pattern':pattern,
8101 'error':'invalid path','response':None})
8102 return Row({'status':200,'response':response,
8103 'pattern':pattern,'count':count})
8104 return Row({'status':400,'error':'no matching pattern','response':None})
8105
8106 - def define_table(
8107 self,
8108 tablename,
8109 *fields,
8110 **args
8111 ):
8112 if not fields and 'fields' in args:
8113 fields = args.get('fields',())
8114 if not isinstance(tablename, str):
8115 if isinstance(tablename, unicode):
8116 try:
8117 tablename = str(tablename)
8118 except UnicodeEncodeError:
8119 raise SyntaxError("invalid unicode table name")
8120 else:
8121 raise SyntaxError("missing table name")
8122 elif hasattr(self,tablename) or tablename in self.tables:
8123 if not args.get('redefine',False):
8124 raise SyntaxError('table already defined: %s' % tablename)
8125 elif tablename.startswith('_') or hasattr(self,tablename) or \
8126 REGEX_PYTHON_KEYWORDS.match(tablename):
8127 raise SyntaxError('invalid table name: %s' % tablename)
8128 elif self.check_reserved:
8129 self.check_reserved_keyword(tablename)
8130 else:
8131 invalid_args = set(args)-TABLE_ARGS
8132 if invalid_args:
8133 raise SyntaxError('invalid table "%s" attributes: %s' \
8134 % (tablename,invalid_args))
8135 if self._lazy_tables and not tablename in self._LAZY_TABLES:
8136 self._LAZY_TABLES[tablename] = (tablename,fields,args)
8137 table = None
8138 else:
8139 table = self.lazy_define_table(tablename,*fields,**args)
8140 if not tablename in self.tables:
8141 self.tables.append(tablename)
8142 return table
8143
8144 - def lazy_define_table(
8145 self,
8146 tablename,
8147 *fields,
8148 **args
8149 ):
8150 args_get = args.get
8151 common_fields = self._common_fields
8152 if common_fields:
8153 fields = list(fields) + list(common_fields)
8154
8155 table_class = args_get('table_class',Table)
8156 table = table_class(self, tablename, *fields, **args)
8157 table._actual = True
8158 self[tablename] = table
8159
8160 table._create_references()
8161 for field in table:
8162 if field.requires == DEFAULT:
8163 field.requires = sqlhtml_validators(field)
8164
8165 migrate = self._migrate_enabled and args_get('migrate',self._migrate)
8166 if migrate and not self._uri in (None,'None') \
8167 or self._adapter.dbengine=='google:datastore':
8168 fake_migrate = self._fake_migrate_all or \
8169 args_get('fake_migrate',self._fake_migrate)
8170 polymodel = args_get('polymodel',None)
8171 try:
8172 GLOBAL_LOCKER.acquire()
8173 self._lastsql = self._adapter.create_table(
8174 table,migrate=migrate,
8175 fake_migrate=fake_migrate,
8176 polymodel=polymodel)
8177 finally:
8178 GLOBAL_LOCKER.release()
8179 else:
8180 table._dbt = None
8181 on_define = args_get('on_define',None)
8182 if on_define: on_define(table)
8183 return table
8184
8185 - def as_dict(self, flat=False, sanitize=True):
8186 db_uid = uri = None
8187 if not sanitize:
8188 uri, db_uid = (self._uri, self._db_uid)
8189 db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid,
8190 **dict([(k, getattr(self, "_" + k, None))
8191 for k in 'pool_size','folder','db_codec',
8192 'check_reserved','migrate','fake_migrate',
8193 'migrate_enabled','fake_migrate_all',
8194 'decode_credentials','driver_args',
8195 'adapter_args', 'attempts',
8196 'bigint_id','debug','lazy_tables',
8197 'do_connect']))
8198 for table in self:
8199 db_as_dict["tables"].append(table.as_dict(flat=flat,
8200 sanitize=sanitize))
8201 return db_as_dict
8202
8203 - def as_xml(self, sanitize=True):
8208
8209 - def as_json(self, sanitize=True):
8214
8215 - def as_yaml(self, sanitize=True):
8220
8222 try:
8223 return tablename in self.tables
8224 except AttributeError:
8225
8226 return False
8227
8228 has_key = __contains__
8229
8230 - def get(self,key,default=None):
8231 return self.__dict__.get(key,default)
8232
8234 for tablename in self.tables:
8235 yield self[tablename]
8236
8239
8241 if ogetattr(self,'_lazy_tables') and \
8242 key in ogetattr(self,'_LAZY_TABLES'):
8243 tablename, fields, args = self._LAZY_TABLES.pop(key)
8244 return self.lazy_define_table(tablename,*fields,**args)
8245 return ogetattr(self, key)
8246
8248 osetattr(self, str(key), value)
8249
8251 if key[:1]!='_' and key in self:
8252 raise SyntaxError(
8253 'Object %s exists and cannot be redefined' % key)
8254 osetattr(self,key,value)
8255
8256 __delitem__ = object.__delattr__
8257
8259 if hasattr(self,'_uri'):
8260 return '<DAL uri="%s">' % hide_password(self._adapter.uri)
8261 else:
8262 return '<DAL db_uid="%s">' % self._db_uid
8263
8266
8267 - def __call__(self, query=None, ignore_common_filters=None):
8268 if isinstance(query,Table):
8269 query = self._adapter.id_query(query)
8270 elif isinstance(query,Field):
8271 query = query!=None
8272 elif isinstance(query, dict):
8273 icf = query.get("ignore_common_filters")
8274 if icf: ignore_common_filters = icf
8275 return Set(self, query, ignore_common_filters=ignore_common_filters)
8276
8279
8282
8284 self._adapter.close()
8285 if self._db_uid in THREAD_LOCAL.db_instances:
8286 db_group = THREAD_LOCAL.db_instances[self._db_uid]
8287 db_group.remove(self)
8288 if not db_group:
8289 del THREAD_LOCAL.db_instances[self._db_uid]
8290
8291 - def executesql(self, query, placeholders=None, as_dict=False,
8292 fields=None, colnames=None, as_ordered_dict=False):
8293 """
8294 placeholders is optional and will always be None.
8295 If using raw SQL with placeholders, placeholders may be
8296 a sequence of values to be substituted in
8297 or, (if supported by the DB driver), a dictionary with keys
8298 matching named placeholders in your SQL.
8299
8300 Added 2009-12-05 "as_dict" optional argument. Will always be
8301 None when using DAL. If using raw SQL can be set to True and
8302 the results cursor returned by the DB driver will be converted
8303 to a sequence of dictionaries keyed with the db field
8304 names. Tested with SQLite but should work with any database
8305 since the cursor.description used to get field names is part
8306 of the Python dbi 2.0 specs. Results returned with
8307 as_dict=True are the same as those returned when applying
8308 .to_list() to a DAL query. If "as_ordered_dict"=True the
8309 behaviour is the same as when "as_dict"=True with the keys
8310 (field names) guaranteed to be in the same order as returned
8311 by the select name executed on the database.
8312
8313 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
8314
8315 Added 2012-08-24 "fields" and "colnames" optional arguments. If either
8316 is provided, the results cursor returned by the DB driver will be
8317 converted to a DAL Rows object using the db._adapter.parse() method.
8318
8319 The "fields" argument is a list of DAL Field objects that match the
8320 fields returned from the DB. The Field objects should be part of one or
8321 more Table objects defined on the DAL object. The "fields" list can
8322 include one or more DAL Table objects in addition to or instead of
8323 including Field objects, or it can be just a single table (not in a
8324 list). In that case, the Field objects will be extracted from the
8325 table(s).
8326
8327 Instead of specifying the "fields" argument, the "colnames" argument
8328 can be specified as a list of field names in tablename.fieldname format.
8329 Again, these should represent tables and fields defined on the DAL
8330 object.
8331
8332 It is also possible to specify both "fields" and the associated
8333 "colnames". In that case, "fields" can also include DAL Expression
8334 objects in addition to Field objects. For Field objects in "fields",
8335 the associated "colnames" must still be in tablename.fieldname format.
8336 For Expression objects in "fields", the associated "colnames" can
8337 be any arbitrary labels.
8338
8339 Note, the DAL Table objects referred to by "fields" or "colnames" can
8340 be dummy tables and do not have to represent any real tables in the
8341 database. Also, note that the "fields" and "colnames" must be in the
8342 same order as the fields in the results cursor returned from the DB.
8343
8344 """
8345 adapter = self._adapter
8346 if placeholders:
8347 adapter.execute(query, placeholders)
8348 else:
8349 adapter.execute(query)
8350 if as_dict or as_ordered_dict:
8351 if not hasattr(adapter.cursor,'description'):
8352 raise RuntimeError("database does not support executesql(...,as_dict=True)")
8353
8354
8355
8356 columns = adapter.cursor.description
8357
8358 fields = colnames or [f[0] for f in columns]
8359
8360 data = adapter._fetchall()
8361
8362
8363 if as_ordered_dict:
8364 _dict = OrderedDict
8365 else:
8366 _dict = dict
8367 return [_dict(zip(fields,row)) for row in data]
8368 try:
8369 data = adapter._fetchall()
8370 except:
8371 return None
8372 if fields or colnames:
8373 fields = [] if fields is None else fields
8374 if not isinstance(fields, list):
8375 fields = [fields]
8376 extracted_fields = []
8377 for field in fields:
8378 if isinstance(field, Table):
8379 extracted_fields.extend([f for f in field])
8380 else:
8381 extracted_fields.append(field)
8382 if not colnames:
8383 colnames = ['%s.%s' % (f.tablename, f.name)
8384 for f in extracted_fields]
8385 data = adapter.parse(
8386 data, fields=extracted_fields, colnames=colnames)
8387 return data
8388
8390 for table in self:
8391 table._referenced_by = [field for field in table._referenced_by
8392 if not field.table==thistable]
8393
8395 step = long(kwargs.get('max_fetch_rows,',500))
8396 write_colnames = kwargs['write_colnames'] = \
8397 kwargs.get("write_colnames", True)
8398 for table in self.tables:
8399 ofile.write('TABLE %s\r\n' % table)
8400 query = self._adapter.id_query(self[table])
8401 nrows = self(query).count()
8402 kwargs['write_colnames'] = write_colnames
8403 for k in range(0,nrows,step):
8404 self(query).select(limitby=(k,k+step)).export_to_csv_file(
8405 ofile, *args, **kwargs)
8406 kwargs['write_colnames'] = False
8407 ofile.write('\r\n\r\n')
8408 ofile.write('END')
8409
8410 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
8411 unique='uuid', map_tablenames=None,
8412 ignore_missing_tables=False,
8413 *args, **kwargs):
8414
8415 id_offset = {}
8416 map_tablenames = map_tablenames or {}
8417 for line in ifile:
8418 line = line.strip()
8419 if not line:
8420 continue
8421 elif line == 'END':
8422 return
8423 elif not line.startswith('TABLE ') or \
8424 not line[6:] in self.tables:
8425 raise SyntaxError('invalid file format')
8426 else:
8427 tablename = line[6:]
8428 tablename = map_tablenames.get(tablename,tablename)
8429 if tablename is not None and tablename in self.tables:
8430 self[tablename].import_from_csv_file(
8431 ifile, id_map, null, unique, id_offset,
8432 *args, **kwargs)
8433 elif tablename is None or ignore_missing_tables:
8434
8435 for line in ifile:
8436 if not line.strip():
8437 break
8438 else:
8439 raise RuntimeError("Unable to import table that does not exist.\nTry db.import_from_csv_file(..., map_tablenames={'table':'othertable'},ignore_missing_tables=True)")
8440
8443 return DAL('<zombie>',db_uid=db_uid)
8444
8447
8448 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
8451 """
8452 Helper class providing a comma-separated string having all the field names
8453 (prefixed by table name and '.')
8454
8455 normally only called from within gluon.sql
8456 """
8457
8460
8462 return ', '.join([str(field) for field in self._table])
8463
8466
8468 if not self._record:
8469 self._record = self._table[long(self)]
8470 if not self._record:
8471 raise RuntimeError(
8472 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, long(self)))
8473
8475 if key == 'id':
8476 return long(self)
8477 if key in self._table:
8478 self.__allocate()
8479 if self._record:
8480 return self._record.get(key,None)
8481 else:
8482 return None
8483
8484 - def get(self, key, default=None):
8486
8493
8495 if key == 'id':
8496 return long(self)
8497 self.__allocate()
8498 return self._record.get(key, None)
8499
8501 self.__allocate()
8502 self._record[key] = value
8503
8506 return marshal.loads(data)
8507
8509 try:
8510 marshal_dump = marshal.dumps(long(data))
8511 except AttributeError:
8512 marshal_dump = 'i%s' % struct.pack('<i', long(data))
8513 return (Reference_unpickler, (marshal_dump,))
8514
8515 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
8525 def _decorated(f):
8526 instance = self.table
8527 import types
8528 method = types.MethodType(f, instance, instance.__class__)
8529 name = method_name or f.func_name
8530 setattr(instance, name, method)
8531 return f
8532 return _decorated
8533
8535
8536 """
8537 an instance of this class represents a database table
8538
8539 Example::
8540
8541 db = DAL(...)
8542 db.define_table('users', Field('name'))
8543 db.users.insert(name='me') # print db.users._insert(...) to see SQL
8544 db.users.drop()
8545 """
8546
8547 - def __init__(
8548 self,
8549 db,
8550 tablename,
8551 *fields,
8552 **args
8553 ):
8554 """
8555 Initializes the table and performs checking on the provided fields.
8556
8557 Each table will have automatically an 'id'.
8558
8559 If a field is of type Table, the fields (excluding 'id') from that table
8560 will be used instead.
8561
8562 :raises SyntaxError: when a supplied field is of incorrect type.
8563 """
8564 self._actual = False
8565 self._tablename = tablename
8566 self._ot = None
8567 self._rname = args.get('rname')
8568 if not self._rname:
8569 self._sequence_name = args.get('sequence_name') or \
8570 db and db._adapter.sequence_name(tablename)
8571 else:
8572 tb = self._rname[1:-1]
8573 self._sequence_name = args.get('sequence_name') or \
8574 db and db._adapter.sequence_name(tb)
8575 self._trigger_name = args.get('trigger_name') or \
8576 db and db._adapter.trigger_name(tablename)
8577 self._common_filter = args.get('common_filter')
8578 self._format = args.get('format')
8579 self._singular = args.get(
8580 'singular',tablename.replace('_',' ').capitalize())
8581 self._plural = args.get(
8582 'plural',pluralize(self._singular.lower()).capitalize())
8583
8584 if 'primarykey' in args and args['primarykey'] is not None:
8585 self._primarykey = args.get('primarykey')
8586
8587 self._before_insert = []
8588 self._before_update = [Set.delete_uploaded_files]
8589 self._before_delete = [Set.delete_uploaded_files]
8590 self._after_insert = []
8591 self._after_update = []
8592 self._after_delete = []
8593
8594 self.add_method = MethodAdder(self)
8595
8596 fieldnames,newfields=set(),[]
8597 _primarykey = getattr(self, '_primarykey', None)
8598 if _primarykey is not None:
8599 if not isinstance(_primarykey, list):
8600 raise SyntaxError(
8601 "primarykey must be a list of fields from table '%s'" \
8602 % tablename)
8603 if len(_primarykey)==1:
8604 self._id = [f for f in fields if isinstance(f,Field) \
8605 and f.name==_primarykey[0]][0]
8606 elif not [f for f in fields if (isinstance(f,Field) and
8607 f.type=='id') or (isinstance(f, dict) and
8608 f.get("type", None)=="id")]:
8609 field = Field('id', 'id')
8610 newfields.append(field)
8611 fieldnames.add('id')
8612 self._id = field
8613 virtual_fields = []
8614 def include_new(field):
8615 newfields.append(field)
8616 fieldnames.add(field.name)
8617 if field.type=='id':
8618 self._id = field
8619 for field in fields:
8620 if isinstance(field, (FieldMethod, FieldVirtual)):
8621 virtual_fields.append(field)
8622 elif isinstance(field, Field) and not field.name in fieldnames:
8623 if field.db is not None:
8624 field = copy.copy(field)
8625 include_new(field)
8626 elif isinstance(field, dict) and not field['fieldname'] in fieldnames:
8627 include_new(Field(**field))
8628 elif isinstance(field, Table):
8629 table = field
8630 for field in table:
8631 if not field.name in fieldnames and not field.type=='id':
8632 t2 = not table._actual and self._tablename
8633 include_new(field.clone(point_self_references_to=t2))
8634 elif not isinstance(field, (Field, Table)):
8635 raise SyntaxError(
8636 'define_table argument is not a Field or Table: %s' % field)
8637 fields = newfields
8638 self._db = db
8639 tablename = tablename
8640 self._fields = SQLCallableList()
8641 self.virtualfields = []
8642 fields = list(fields)
8643
8644 if db and db._adapter.uploads_in_blob==True:
8645 uploadfields = [f.name for f in fields if f.type=='blob']
8646 for field in fields:
8647 fn = field.uploadfield
8648 if isinstance(field, Field) and field.type == 'upload'\
8649 and fn is True:
8650 fn = field.uploadfield = '%s_blob' % field.name
8651 if isinstance(fn,str) and not fn in uploadfields:
8652 fields.append(Field(fn,'blob',default='',
8653 writable=False,readable=False))
8654
8655 lower_fieldnames = set()
8656 reserved = dir(Table) + ['fields']
8657 if (db and db.check_reserved):
8658 check_reserved = db.check_reserved_keyword
8659 else:
8660 def check_reserved(field_name):
8661 if field_name in reserved:
8662 raise SyntaxError("field name %s not allowed" % field_name)
8663 for field in fields:
8664 field_name = field.name
8665 check_reserved(field_name)
8666 fn_lower = field_name.lower()
8667 if fn_lower in lower_fieldnames:
8668 raise SyntaxError("duplicate field %s in table %s" \
8669 % (field_name, tablename))
8670 else:
8671 lower_fieldnames.add(fn_lower)
8672
8673 self.fields.append(field_name)
8674 self[field_name] = field
8675 if field.type == 'id':
8676 self['id'] = field
8677 field.tablename = field._tablename = tablename
8678 field.table = field._table = self
8679 field.db = field._db = db
8680 self.ALL = SQLALL(self)
8681
8682 if _primarykey is not None:
8683 for k in _primarykey:
8684 if k not in self.fields:
8685 raise SyntaxError(
8686 "primarykey must be a list of fields from table '%s " % tablename)
8687 else:
8688 self[k].notnull = True
8689 for field in virtual_fields:
8690 self[field.name] = field
8691
8692 @property
8695
8696 - def update(self,*args,**kwargs):
8697 raise RuntimeError("Syntax Not Supported")
8698
8699 - def _enable_record_versioning(self,
8700 archive_db=None,
8701 archive_name = '%(tablename)s_archive',
8702 is_active = 'is_active',
8703 current_record = 'current_record',
8704 current_record_label = None):
8705 db = self._db
8706 archive_db = archive_db or db
8707 archive_name = archive_name % dict(tablename=self._tablename)
8708 if archive_name in archive_db.tables():
8709 return
8710 fieldnames = self.fields()
8711 same_db = archive_db is db
8712 field_type = self if same_db else 'bigint'
8713 clones = []
8714 for field in self:
8715 nfk = same_db or not field.type.startswith('reference')
8716 clones.append(field.clone(
8717 unique=False, type=field.type if nfk else 'bigint'))
8718 archive_db.define_table(
8719 archive_name,
8720 Field(current_record,field_type,label=current_record_label),
8721 *clones,**dict(format=self._format))
8722
8723 self._before_update.append(
8724 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record:
8725 archive_record(qset,fs,db[an],cn))
8726 if is_active and is_active in fieldnames:
8727 self._before_delete.append(
8728 lambda qset: qset.update(is_active=False))
8729 newquery = lambda query, t=self, name=self._tablename: \
8730 reduce(AND,[db[tn].is_active == True
8731 for tn in db._adapter.tables(query)
8732 if tn==name or getattr(db[tn],'_ot',None)==name])
8733 query = self._common_filter
8734 if query:
8735 newquery = query & newquery
8736 self._common_filter = newquery
8737
8745
8747 db = self._db
8748 pr = db._pending_references
8749 self._referenced_by = []
8750 self._references = []
8751 for field in self:
8752 fieldname = field.name
8753 field_type = field.type
8754 if isinstance(field_type,str) and field_type[:10] == 'reference ':
8755 ref = field_type[10:].strip()
8756 if not ref:
8757 SyntaxError('Table: reference to nothing: %s' %ref)
8758 if '.' in ref:
8759 rtablename, throw_it,rfieldname = ref.partition('.')
8760 else:
8761 rtablename, rfieldname = ref, None
8762 if not rtablename in db:
8763 pr[rtablename] = pr.get(rtablename,[]) + [field]
8764 continue
8765 rtable = db[rtablename]
8766 if rfieldname:
8767 if not hasattr(rtable,'_primarykey'):
8768 raise SyntaxError(
8769 'keyed tables can only reference other keyed tables (for now)')
8770 if rfieldname not in rtable.fields:
8771 raise SyntaxError(
8772 "invalid field '%s' for referenced table '%s' in table '%s'" \
8773 % (rfieldname, rtablename, self._tablename))
8774 rfield = rtable[rfieldname]
8775 else:
8776 rfield = rtable._id
8777 rtable._referenced_by.append(field)
8778 field.referent = rfield
8779 self._references.append(field)
8780 else:
8781 field.referent = None
8782 if self._tablename in pr:
8783 referees = pr.pop(self._tablename)
8784 for referee in referees:
8785 self._referenced_by.append(referee)
8786
8787
8789 return dict([(k, v) for (k, v) in record.iteritems() if k
8790 in self.fields and (self[k].type!='id' or id)])
8791
8793 """ for keyed table only """
8794 query = None
8795 for k,v in key.iteritems():
8796 if k in self._primarykey:
8797 if query:
8798 query = query & (self[k] == v)
8799 else:
8800 query = (self[k] == v)
8801 else:
8802 raise SyntaxError(
8803 'Field %s is not part of the primary key of %s' % \
8804 (k,self._tablename))
8805 return query
8806
8808 if not key:
8809 return None
8810 elif isinstance(key, dict):
8811 """ for keyed table """
8812 query = self._build_query(key)
8813 return self._db(query).select(limitby=(0,1), orderby_on_limitby=False).first()
8814 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key):
8815 return self._db(self._id == key).select(limitby=(0,1), orderby_on_limitby=False).first()
8816 elif key:
8817 return ogetattr(self, str(key))
8818
8820 for_update = kwargs.get('_for_update',False)
8821 if '_for_update' in kwargs: del kwargs['_for_update']
8822
8823 orderby = kwargs.get('_orderby',None)
8824 if '_orderby' in kwargs: del kwargs['_orderby']
8825
8826 if not key is DEFAULT:
8827 if isinstance(key, Query):
8828 record = self._db(key).select(
8829 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8830 elif not str(key).isdigit():
8831 record = None
8832 else:
8833 record = self._db(self._id == key).select(
8834 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8835 if record:
8836 for k,v in kwargs.iteritems():
8837 if record[k]!=v: return None
8838 return record
8839 elif kwargs:
8840 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
8841 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8842 else:
8843 return None
8844
8846 if isinstance(key, dict) and isinstance(value, dict):
8847 """ option for keyed table """
8848 if set(key.keys()) == set(self._primarykey):
8849 value = self._filter_fields(value)
8850 kv = {}
8851 kv.update(value)
8852 kv.update(key)
8853 if not self.insert(**kv):
8854 query = self._build_query(key)
8855 self._db(query).update(**self._filter_fields(value))
8856 else:
8857 raise SyntaxError(
8858 'key must have all fields from primary key: %s'%\
8859 (self._primarykey))
8860 elif str(key).isdigit():
8861 if key == 0:
8862 self.insert(**self._filter_fields(value))
8863 elif self._db(self._id == key)\
8864 .update(**self._filter_fields(value)) is None:
8865 raise SyntaxError('No such record: %s' % key)
8866 else:
8867 if isinstance(key, dict):
8868 raise SyntaxError(
8869 'value must be a dictionary: %s' % value)
8870 osetattr(self, str(key), value)
8871
8872 __getattr__ = __getitem__
8873
8875 if key[:1]!='_' and key in self:
8876 raise SyntaxError('Object exists and cannot be redefined: %s' % key)
8877 osetattr(self,key,value)
8878
8880 if isinstance(key, dict):
8881 query = self._build_query(key)
8882 if not self._db(query).delete():
8883 raise SyntaxError('No such record: %s' % key)
8884 elif not str(key).isdigit() or \
8885 not self._db(self._id == key).delete():
8886 raise SyntaxError('No such record: %s' % key)
8887
8889 return hasattr(self,key)
8890
8891 has_key = __contains__
8892
8894 return self.__dict__.items()
8895
8897 for fieldname in self.fields:
8898 yield self[fieldname]
8899
8902
8903
8905 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
8906
8908 if self._ot is not None:
8909 ot = self._ot
8910 if 'Oracle' in str(type(self._db._adapter)):
8911 return '%s %s' % (ot, self._tablename)
8912 return '%s AS %s' % (ot, self._tablename)
8913 return self._tablename
8914
8915 - def _drop(self, mode = ''):
8916 return self._db._adapter._drop(self, mode)
8917
8918 - def drop(self, mode = ''):
8919 return self._db._adapter.drop(self,mode)
8920
8921 - def _listify(self,fields,update=False):
8922 new_fields = {}
8923
8924
8925 for name in fields:
8926 if not name in self.fields:
8927 if name != 'id':
8928 raise SyntaxError(
8929 'Field %s does not belong to the table' % name)
8930 else:
8931 field = self[name]
8932 value = fields[name]
8933 if field.filter_in:
8934 value = field.filter_in(value)
8935 new_fields[name] = (field,value)
8936
8937
8938 to_compute = []
8939 for ofield in self:
8940 name = ofield.name
8941 if not name in new_fields:
8942
8943 if ofield.compute:
8944 to_compute.append((name,ofield))
8945
8946 elif not update and not ofield.default is None:
8947 value = ofield.default
8948 fields[name] = value
8949 new_fields[name] = (ofield,value)
8950
8951 elif update and not ofield.update is None:
8952 value = ofield.update
8953 fields[name] = value
8954 new_fields[name] = (ofield,value)
8955
8956 elif not update and ofield.required:
8957 raise RuntimeError(
8958 'Table: missing required field: %s' % name)
8959
8960 if to_compute:
8961 row = Row(fields)
8962 for name,ofield in to_compute:
8963
8964 try:
8965 row[name] = new_value = ofield.compute(row)
8966 new_fields[name] = (ofield, new_value)
8967 except (KeyError, AttributeError):
8968
8969 if ofield.required:
8970 raise SyntaxError('unable to compute field: %s' % name)
8971 return new_fields.values()
8972
8974 for field in self:
8975 if field.type=='upload' and field.name in fields:
8976 value = fields[field.name]
8977 if value is not None and not isinstance(value,str):
8978 if hasattr(value,'file') and hasattr(value,'filename'):
8979 new_name = field.store(value.file,filename=value.filename)
8980 elif hasattr(value,'read') and hasattr(value,'name'):
8981 new_name = field.store(value,filename=value.name)
8982 else:
8983 raise RuntimeError("Unable to handle upload")
8984 fields[field.name] = new_name
8985
8987 "If there are no fields/values specified, return table defaults"
8988 if not fields:
8989 fields = {}
8990 for field in self:
8991 if field.type != "id":
8992 fields[field.name] = field.default
8993 return fields
8994
8998
9008
9024
9050
9052 if _key is DEFAULT:
9053 record = self(**values)
9054 elif isinstance(_key,dict):
9055 record = self(**_key)
9056 else:
9057 record = self(_key)
9058 if record:
9059 record.update_record(**values)
9060 newid = None
9061 else:
9062 newid = self.insert(**values)
9063 return newid
9064
9066 """
9067 here items is a list of dictionaries
9068 """
9069 items = [self._listify(item) for item in items]
9070 if any(f(item) for item in items for f in self._before_insert):return 0
9071 ret = self._db._adapter.bulk_insert(self,items)
9072 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert]
9073 return ret
9074
9076 return self._db._adapter._truncate(self, mode)
9077
9079 return self._db._adapter.truncate(self, mode)
9080
9081 - def import_from_csv_file(
9082 self,
9083 csvfile,
9084 id_map=None,
9085 null='<NULL>',
9086 unique='uuid',
9087 id_offset=None,
9088 *args, **kwargs
9089 ):
9090 """
9091 Import records from csv file.
9092 Column headers must have same names as table fields.
9093 Field 'id' is ignored.
9094 If column names read 'table.file' the 'table.' prefix is ignored.
9095 'unique' argument is a field which must be unique
9096 (typically a uuid field)
9097 'restore' argument is default False;
9098 if set True will remove old values in table first.
9099 'id_map' if set to None will not map ids.
9100 The import will keep the id numbers in the restored table.
9101 This assumes that there is an field of type id that
9102 is integer and in incrementing order.
9103 Will keep the id numbers in restored table.
9104 """
9105
9106 delimiter = kwargs.get('delimiter', ',')
9107 quotechar = kwargs.get('quotechar', '"')
9108 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
9109 restore = kwargs.get('restore', False)
9110 if restore:
9111 self._db[self].truncate()
9112
9113 reader = csv.reader(csvfile, delimiter=delimiter,
9114 quotechar=quotechar, quoting=quoting)
9115 colnames = None
9116 if isinstance(id_map, dict):
9117 if not self._tablename in id_map:
9118 id_map[self._tablename] = {}
9119 id_map_self = id_map[self._tablename]
9120
9121 def fix(field, value, id_map, id_offset):
9122 list_reference_s='list:reference'
9123 if value == null:
9124 value = None
9125 elif field.type=='blob':
9126 value = base64.b64decode(value)
9127 elif field.type=='double' or field.type=='float':
9128 if not value.strip():
9129 value = None
9130 else:
9131 value = float(value)
9132 elif field.type in ('integer','bigint'):
9133 if not value.strip():
9134 value = None
9135 else:
9136 value = long(value)
9137 elif field.type.startswith('list:string'):
9138 value = bar_decode_string(value)
9139 elif field.type.startswith(list_reference_s):
9140 ref_table = field.type[len(list_reference_s):].strip()
9141 if id_map is not None:
9142 value = [id_map[ref_table][long(v)] \
9143 for v in bar_decode_string(value)]
9144 else:
9145 value = [v for v in bar_decode_string(value)]
9146 elif field.type.startswith('list:'):
9147 value = bar_decode_integer(value)
9148 elif id_map and field.type.startswith('reference'):
9149 try:
9150 value = id_map[field.type[9:].strip()][long(value)]
9151 except KeyError:
9152 pass
9153 elif id_offset and field.type.startswith('reference'):
9154 try:
9155 value = id_offset[field.type[9:].strip()]+long(value)
9156 except KeyError:
9157 pass
9158 return (field.name, value)
9159
9160 def is_id(colname):
9161 if colname in self:
9162 return self[colname].type == 'id'
9163 else:
9164 return False
9165
9166 first = True
9167 unique_idx = None
9168 for lineno, line in enumerate(reader):
9169 if not line:
9170 break
9171 if not colnames:
9172
9173 colnames = [x.split('.',1)[-1] for x in line][:len(line)]
9174 cols, cid = [], None
9175 for i,colname in enumerate(colnames):
9176 if is_id(colname):
9177 cid = i
9178 elif colname in self.fields:
9179 cols.append((i,self[colname]))
9180 if colname == unique:
9181 unique_idx = i
9182 else:
9183
9184 items = []
9185 for i, field in cols:
9186 try:
9187 items.append(fix(field, line[i], id_map, id_offset))
9188 except ValueError:
9189 raise RuntimeError("Unable to parse line:%s field:%s value:'%s'"
9190 % (lineno+1,field,line[i]))
9191
9192 if not (id_map or cid is None or id_offset is None or unique_idx):
9193 csv_id = long(line[cid])
9194 curr_id = self.insert(**dict(items))
9195 if first:
9196 first = False
9197
9198
9199
9200 id_offset[self._tablename] = (curr_id-csv_id) \
9201 if curr_id>csv_id else 0
9202
9203 while curr_id<csv_id+id_offset[self._tablename]:
9204 self._db(self._db[self][colnames[cid]] == curr_id).delete()
9205 curr_id = self.insert(**dict(items))
9206
9207
9208 elif not unique_idx:
9209 new_id = self.insert(**dict(items))
9210 else:
9211 unique_value = line[unique_idx]
9212 query = self._db[self][unique] == unique_value
9213 record = self._db(query).select().first()
9214 if record:
9215 record.update_record(**dict(items))
9216 new_id = record[self._id.name]
9217 else:
9218 new_id = self.insert(**dict(items))
9219 if id_map and cid is not None:
9220 id_map_self[long(line[cid])] = new_id
9221
9222 - def as_dict(self, flat=False, sanitize=True):
9234
9235 - def as_xml(self, sanitize=True):
9240
9241 - def as_json(self, sanitize=True):
9246
9247 - def as_yaml(self, sanitize=True):
9252
9255
9256 - def on(self, query):
9257 return Expression(self._db,self._db._adapter.ON,self,query)
9258
9260 tablenames = qset.db._adapter.tables(qset.query)
9261 if len(tablenames)!=1: raise RuntimeError("cannot update join")
9262 table = qset.db[tablenames[0]]
9263 for row in qset.select():
9264 fields = archive_table._filter_fields(row)
9265 fields[current_record] = row.id
9266 archive_table.insert(**fields)
9267 return False
9268
9272
9273 - def __init__(
9274 self,
9275 db,
9276 op,
9277 first=None,
9278 second=None,
9279 type=None,
9280 **optional_args
9281 ):
9282
9283 self.db = db
9284 self.op = op
9285 self.first = first
9286 self.second = second
9287 self._table = getattr(first,'_table',None)
9288
9289 if not type and first and hasattr(first,'type'):
9290 self.type = first.type
9291 else:
9292 self.type = type
9293 self.optional_args = optional_args
9294
9298
9302
9306
9310
9314
9318
9322
9326
9330
9334
9338
9342
9346
9350
9354
9358
9362
9366
9368 db = self.db
9369 if start < 0:
9370 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
9371 else:
9372 pos0 = start + 1
9373
9374 if stop < 0:
9375 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
9376 elif stop == sys.maxint:
9377 length = self.len()
9378 else:
9379 length = '(%s - %s)' % (stop + 1, pos0)
9380 return Expression(db,db._adapter.SUBSTRING,
9381 self, (pos0, length), self.type)
9382
9384 return self[i:i + 1]
9385
9387 return self.db._adapter.expand(self,self.type)
9388
9390 db = self.db
9391 return Expression(db,db._adapter.COMMA,self,other,self.type)
9392
9398
9402
9404 db = self.db
9405 if self.type in ('integer','bigint'):
9406 result_type = 'integer'
9407 elif self.type in ['date','time','datetime','double','float']:
9408 result_type = 'double'
9409 elif self.type.startswith('decimal('):
9410 result_type = self.type
9411 else:
9412 raise SyntaxError("subtraction operation not supported for type")
9413 return Expression(db,db._adapter.SUB,self,other,result_type)
9414
9418
9422
9426
9430
9434
9438
9442
9446
9450
9451 - def like(self, value, case_sensitive=False):
9452 db = self.db
9453 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE
9454 return Query(db, op, self, value)
9455
9459
9460 - def belongs(self, *value, **kwattr):
9461 """
9462 Accepts the following inputs:
9463 field.belongs(1,2)
9464 field.belongs((1,2))
9465 field.belongs(query)
9466
9467 Does NOT accept:
9468 field.belongs(1)
9469 """
9470 db = self.db
9471 if len(value) == 1:
9472 value = value[0]
9473 if isinstance(value,Query):
9474 value = db(value)._select(value.first._table._id)
9475 elif not isinstance(value, basestring):
9476 value = set(value)
9477 if kwattr.get('null') and None in value:
9478 value.remove(None)
9479 return (self == None) | Query(db, db._adapter.BELONGS, self, value)
9480 return Query(db, db._adapter.BELONGS, self, value)
9481
9483 db = self.db
9484 if not self.type in ('string', 'text', 'json', 'upload'):
9485 raise SyntaxError("startswith used with incompatible field type")
9486 return Query(db, db._adapter.STARTSWITH, self, value)
9487
9489 db = self.db
9490 if not self.type in ('string', 'text', 'json', 'upload'):
9491 raise SyntaxError("endswith used with incompatible field type")
9492 return Query(db, db._adapter.ENDSWITH, self, value)
9493
9494 - def contains(self, value, all=False, case_sensitive=False):
9495 """
9496 The case_sensitive parameters is only useful for PostgreSQL
9497 For other RDMBs it is ignored and contains is always case in-sensitive
9498 For MongoDB and GAE contains is always case sensitive
9499 """
9500 db = self.db
9501 if isinstance(value,(list, tuple)):
9502 subqueries = [self.contains(str(v).strip(),case_sensitive=case_sensitive)
9503 for v in value if str(v).strip()]
9504 if not subqueries:
9505 return self.contains('')
9506 else:
9507 return reduce(all and AND or OR,subqueries)
9508 if not self.type in ('string', 'text', 'json', 'upload') and not self.type.startswith('list:'):
9509 raise SyntaxError("contains used with incompatible field type")
9510 return Query(db, db._adapter.CONTAINS, self, value, case_sensitive=case_sensitive)
9511
9515
9516
9517
9518 - def st_asgeojson(self, precision=15, options=0, version=1):
9522
9523 - def st_astext(self):
9524 db = self.db
9525 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
9526
9530
9534
9538
9542
9543
9544
9548
9552
9556
9560
9564
9568
9573 """
9574 allows defining of custom SQL types
9575
9576 Example::
9577
9578 decimal = SQLCustomType(
9579 type ='double',
9580 native ='integer',
9581 encoder =(lambda x: int(float(x) * 100)),
9582 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
9583 )
9584
9585 db.define_table(
9586 'example',
9587 Field('value', type=decimal)
9588 )
9589
9590 :param type: the web2py type (default = 'string')
9591 :param native: the backend type
9592 :param encoder: how to encode the value to store it in the backend
9593 :param decoder: how to decode the value retrieved from the backend
9594 :param validator: what validators to use ( default = None, will use the
9595 default validator for type)
9596 """
9597
9598 - def __init__(
9599 self,
9600 type='string',
9601 native=None,
9602 encoder=None,
9603 decoder=None,
9604 validator=None,
9605 _class=None,
9606 ):
9607
9608 self.type = type
9609 self.native = native
9610 self.encoder = encoder or (lambda x: x)
9611 self.decoder = decoder or (lambda x: x)
9612 self.validator = validator
9613 self._class = _class or type
9614
9616 try:
9617 return self.type.startswith(self, text)
9618 except TypeError:
9619 return False
9620
9623
9626
9629
9631 - def __init__(self, name, f=None, ftype='string',label=None,table_name=None):
9646 return '%s.%s' % (self.tablename, self.name)
9647
9649 - def __init__(self, name, f=None, handler=None):
9653
9655 return ', '.join(str(y) for y in x or [])
9656
9657 -class Field(Expression):
9658
9659 Virtual = FieldVirtual
9660 Method = FieldMethod
9661 Lazy = FieldMethod
9662
9663 """
9664 an instance of this class represents a database field
9665
9666 example::
9667
9668 a = Field(name, 'string', length=32, default=None, required=False,
9669 requires=IS_NOT_EMPTY(), ondelete='CASCADE',
9670 notnull=False, unique=False,
9671 uploadfield=True, widget=None, label=None, comment=None,
9672 uploadfield=True, # True means store on disk,
9673 # 'a_field_name' means store in this field in db
9674 # False means file content will be discarded.
9675 writable=True, readable=True, update=None, authorize=None,
9676 autodelete=False, represent=None, uploadfolder=None,
9677 uploadseparate=False # upload to separate directories by uuid_keys
9678 # first 2 character and tablename.fieldname
9679 # False - old behavior
9680 # True - put uploaded file in
9681 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
9682 # directory)
9683 uploadfs=None # a pyfilesystem where to store upload
9684
9685 to be used as argument of DAL.define_table
9686
9687 allowed field types:
9688 string, boolean, integer, double, text, blob,
9689 date, time, datetime, upload, password
9690
9691 """
9692
9693 - def __init__(
9694 self,
9695 fieldname,
9696 type='string',
9697 length=None,
9698 default=DEFAULT,
9699 required=False,
9700 requires=DEFAULT,
9701 ondelete='CASCADE',
9702 notnull=False,
9703 unique=False,
9704 uploadfield=True,
9705 widget=None,
9706 label=None,
9707 comment=None,
9708 writable=True,
9709 readable=True,
9710 update=None,
9711 authorize=None,
9712 autodelete=False,
9713 represent=None,
9714 uploadfolder=None,
9715 uploadseparate=False,
9716 uploadfs=None,
9717 compute=None,
9718 custom_store=None,
9719 custom_retrieve=None,
9720 custom_retrieve_file_properties=None,
9721 custom_delete=None,
9722 filter_in = None,
9723 filter_out = None,
9724 custom_qualifier = None,
9725 map_none = None,
9726 rname = None
9727 ):
9728 self._db = self.db = None
9729 self.op = None
9730 self.first = None
9731 self.second = None
9732 if isinstance(fieldname, unicode):
9733 try:
9734 fieldname = str(fieldname)
9735 except UnicodeEncodeError:
9736 raise SyntaxError('Field: invalid unicode field name')
9737 self.name = fieldname = cleanup(fieldname)
9738 if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \
9739 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
9740 raise SyntaxError('Field: invalid field name: %s' % fieldname)
9741 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
9742 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512)
9743 self.default = default if default!=DEFAULT else (update or None)
9744 self.required = required
9745 self.ondelete = ondelete.upper()
9746 self.notnull = notnull
9747 self.unique = unique
9748 self.uploadfield = uploadfield
9749 self.uploadfolder = uploadfolder
9750 self.uploadseparate = uploadseparate
9751 self.uploadfs = uploadfs
9752 self.widget = widget
9753 self.comment = comment
9754 self.writable = writable
9755 self.readable = readable
9756 self.update = update
9757 self.authorize = authorize
9758 self.autodelete = autodelete
9759 self.represent = list_represent if \
9760 represent==None and type in ('list:integer','list:string') else represent
9761 self.compute = compute
9762 self.isattachment = True
9763 self.custom_store = custom_store
9764 self.custom_retrieve = custom_retrieve
9765 self.custom_retrieve_file_properties = custom_retrieve_file_properties
9766 self.custom_delete = custom_delete
9767 self.filter_in = filter_in
9768 self.filter_out = filter_out
9769 self.custom_qualifier = custom_qualifier
9770 self.label = label if label!=None else fieldname.replace('_',' ').title()
9771 self.requires = requires if requires!=None else []
9772 self.map_none = map_none
9773 self._rname = rname
9774
9776 self.__dict__.update(*args,**attributes)
9777
9778 - def clone(self,point_self_references_to=False,**args):
9779 field = copy.copy(self)
9780 if point_self_references_to and \
9781 field.type == 'reference %s'+field._tablename:
9782 field.type = 'reference %s' % point_self_references_to
9783 field.__dict__.update(args)
9784 return field
9785
9786 - def store(self, file, filename=None, path=None):
9787 if self.custom_store:
9788 return self.custom_store(file,filename,path)
9789 if isinstance(file, cgi.FieldStorage):
9790 filename = filename or file.filename
9791 file = file.file
9792 elif not filename:
9793 filename = file.name
9794 filename = os.path.basename(filename.replace('/', os.sep)\
9795 .replace('\\', os.sep))
9796 m = REGEX_STORE_PATTERN.search(filename)
9797 extension = m and m.group('e') or 'txt'
9798 uuid_key = web2py_uuid().replace('-', '')[-16:]
9799 encoded_filename = base64.b16encode(filename).lower()
9800 newfilename = '%s.%s.%s.%s' % \
9801 (self._tablename, self.name, uuid_key, encoded_filename)
9802 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
9803 self_uploadfield = self.uploadfield
9804 if isinstance(self_uploadfield,Field):
9805 blob_uploadfield_name = self_uploadfield.uploadfield
9806 keys={self_uploadfield.name: newfilename,
9807 blob_uploadfield_name: file.read()}
9808 self_uploadfield.table.insert(**keys)
9809 elif self_uploadfield == True:
9810 if path:
9811 pass
9812 elif self.uploadfolder:
9813 path = self.uploadfolder
9814 elif self.db._adapter.folder:
9815 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9816 else:
9817 raise RuntimeError(
9818 "you must specify a Field(...,uploadfolder=...)")
9819 if self.uploadseparate:
9820 if self.uploadfs:
9821 raise RuntimeError("not supported")
9822 path = pjoin(path,"%s.%s" %(self._tablename, self.name),
9823 uuid_key[:2])
9824 if not exists(path):
9825 os.makedirs(path)
9826 pathfilename = pjoin(path, newfilename)
9827 if self.uploadfs:
9828 dest_file = self.uploadfs.open(newfilename, 'wb')
9829 else:
9830 dest_file = open(pathfilename, 'wb')
9831 try:
9832 shutil.copyfileobj(file, dest_file)
9833 except IOError:
9834 raise IOError(
9835 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename)
9836 dest_file.close()
9837 return newfilename
9838
9839 - def retrieve(self, name, path=None, nameonly=False):
9840 """
9841 if nameonly==True return (filename, fullfilename) instead of
9842 (filename, stream)
9843 """
9844 self_uploadfield = self.uploadfield
9845 if self.custom_retrieve:
9846 return self.custom_retrieve(name, path)
9847 import http
9848 if self.authorize or isinstance(self_uploadfield, str):
9849 row = self.db(self == name).select().first()
9850 if not row:
9851 raise http.HTTP(404)
9852 if self.authorize and not self.authorize(row):
9853 raise http.HTTP(403)
9854 file_properties = self.retrieve_file_properties(name,path)
9855 filename = file_properties['filename']
9856 if isinstance(self_uploadfield, str):
9857 stream = StringIO.StringIO(row[self_uploadfield] or '')
9858 elif isinstance(self_uploadfield,Field):
9859 blob_uploadfield_name = self_uploadfield.uploadfield
9860 query = self_uploadfield == name
9861 data = self_uploadfield.table(query)[blob_uploadfield_name]
9862 stream = StringIO.StringIO(data)
9863 elif self.uploadfs:
9864
9865 stream = self.uploadfs.open(name, 'rb')
9866 else:
9867
9868
9869
9870 fullname = pjoin(file_properties['path'],name)
9871 if nameonly:
9872 return (filename, fullname)
9873 stream = open(fullname,'rb')
9874 return (filename, stream)
9875
9877 m = REGEX_UPLOAD_PATTERN.match(name)
9878 if not m or not self.isattachment:
9879 raise TypeError('Can\'t retrieve %s file properties' % name)
9880 self_uploadfield = self.uploadfield
9881 if self.custom_retrieve_file_properties:
9882 return self.custom_retrieve_file_properties(name, path)
9883 if m.group('name'):
9884 try:
9885 filename = base64.b16decode(m.group('name'), True)
9886 filename = REGEX_CLEANUP_FN.sub('_', filename)
9887 except (TypeError, AttributeError):
9888 filename = name
9889 else:
9890 filename = name
9891
9892 if isinstance(self_uploadfield, (str, Field)):
9893 return dict(path=None,filename=filename)
9894
9895 if not path:
9896 if self.uploadfolder:
9897 path = self.uploadfolder
9898 else:
9899 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9900 if self.uploadseparate:
9901 t = m.group('table')
9902 f = m.group('field')
9903 u = m.group('uuidkey')
9904 path = pjoin(path,"%s.%s" % (t,f),u[:2])
9905 return dict(path=path,filename=filename)
9906
9907
9923
9935
9936 - def count(self, distinct=None):
9938
9939 - def as_dict(self, flat=False, sanitize=True):
9940 attrs = ('name', 'authorize', 'represent', 'ondelete',
9941 'custom_store', 'autodelete', 'custom_retrieve',
9942 'filter_out', 'uploadseparate', 'widget', 'uploadfs',
9943 'update', 'custom_delete', 'uploadfield', 'uploadfolder',
9944 'custom_qualifier', 'unique', 'writable', 'compute',
9945 'map_none', 'default', 'type', 'required', 'readable',
9946 'requires', 'comment', 'label', 'length', 'notnull',
9947 'custom_retrieve_file_properties', 'filter_in')
9948 serializable = (int, long, basestring, float, tuple,
9949 bool, type(None))
9950
9951 def flatten(obj):
9952 if isinstance(obj, dict):
9953 return dict((flatten(k), flatten(v)) for k, v in
9954 obj.items())
9955 elif isinstance(obj, (tuple, list, set)):
9956 return [flatten(v) for v in obj]
9957 elif isinstance(obj, serializable):
9958 return obj
9959 elif isinstance(obj, (datetime.datetime,
9960 datetime.date, datetime.time)):
9961 return str(obj)
9962 else:
9963 return None
9964
9965 d = dict()
9966 if not (sanitize and not (self.readable or self.writable)):
9967 for attr in attrs:
9968 if flat:
9969 d.update({attr: flatten(getattr(self, attr))})
9970 else:
9971 d.update({attr: getattr(self, attr)})
9972 d["fieldname"] = d.pop("name")
9973 return d
9974
9975 - def as_xml(self, sanitize=True):
9982
9983 - def as_json(self, sanitize=True):
9990
9991 - def as_yaml(self, sanitize=True):
9997
10000
10002 try:
10003 return '%s.%s' % (self.tablename, self.name)
10004 except:
10005 return '<no table>.%s' % self.name
10006
10007
10008 -class Query(object):
10009
10010 """
10011 a query object necessary to define a set.
10012 it can be stored or can be passed to DAL.__call__() to obtain a Set
10013
10014 Example::
10015
10016 query = db.users.name=='Max'
10017 set = db(query)
10018 records = set.select()
10019
10020 """
10021
10022 - def __init__(
10023 self,
10024 db,
10025 op,
10026 first=None,
10027 second=None,
10028 ignore_common_filters = False,
10029 **optional_args
10030 ):
10031 self.db = self._db = db
10032 self.op = op
10033 self.first = first
10034 self.second = second
10035 self.ignore_common_filters = ignore_common_filters
10036 self.optional_args = optional_args
10037
10040
10042 return str(self.db._adapter.expand(self))
10043
10046
10047 __rand__ = __and__
10048
10050 return Query(self.db,self.db._adapter.OR,self,other)
10051
10052 __ror__ = __or__
10053
10055 if self.op==self.db._adapter.NOT:
10056 return self.first
10057 return Query(self.db,self.db._adapter.NOT,self)
10058
10060 return repr(self) == repr(other)
10061
10063 return not (self == other)
10064
10065 - def case(self,t=1,f=0):
10066 return self.db._adapter.CASE(self,t,f)
10067
10068 - def as_dict(self, flat=False, sanitize=True):
10069 """Experimental stuff
10070
10071 This allows to return a plain dictionary with the basic
10072 query representation. Can be used with json/xml services
10073 for client-side db I/O
10074
10075 Example:
10076 >>> q = db.auth_user.id != 0
10077 >>> q.as_dict(flat=True)
10078 {"op": "NE", "first":{"tablename": "auth_user",
10079 "fieldname": "id"},
10080 "second":0}
10081 """
10082
10083 SERIALIZABLE_TYPES = (tuple, dict, set, list, int, long, float,
10084 basestring, type(None), bool)
10085 def loop(d):
10086 newd = dict()
10087 for k, v in d.items():
10088 if k in ("first", "second"):
10089 if isinstance(v, self.__class__):
10090 newd[k] = loop(v.__dict__)
10091 elif isinstance(v, Field):
10092 newd[k] = {"tablename": v._tablename,
10093 "fieldname": v.name}
10094 elif isinstance(v, Expression):
10095 newd[k] = loop(v.__dict__)
10096 elif isinstance(v, SERIALIZABLE_TYPES):
10097 newd[k] = v
10098 elif isinstance(v, (datetime.date,
10099 datetime.time,
10100 datetime.datetime)):
10101 newd[k] = unicode(v)
10102 elif k == "op":
10103 if callable(v):
10104 newd[k] = v.__name__
10105 elif isinstance(v, basestring):
10106 newd[k] = v
10107 else: pass
10108 elif isinstance(v, SERIALIZABLE_TYPES):
10109 if isinstance(v, dict):
10110 newd[k] = loop(v)
10111 else: newd[k] = v
10112 return newd
10113
10114 if flat:
10115 return loop(self.__dict__)
10116 else: return self.__dict__
10117
10118
10119 - def as_xml(self, sanitize=True):
10126
10127 - def as_json(self, sanitize=True):
10134
10136 if not orderby:
10137 return None
10138 orderby2 = orderby[0]
10139 for item in orderby[1:]:
10140 orderby2 = orderby2 | item
10141 return orderby2
10142
10144 return (query and hasattr(query,'ignore_common_filters') and \
10145 not query.ignore_common_filters)
10146
10147 -class Set(object):
10148
10149 """
10150 a Set represents a set of records in the database,
10151 the records are identified by the query=Query(...) object.
10152 normally the Set is generated by DAL.__call__(Query(...))
10153
10154 given a set, for example
10155 set = db(db.users.name=='Max')
10156 you can:
10157 set.update(db.users.name='Massimo')
10158 set.delete() # all elements in the set
10159 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
10160 and take subsets:
10161 subset = set(db.users.id<5)
10162 """
10163
10164 - def __init__(self, db, query, ignore_common_filters = None):
10165 self.db = db
10166 self._db = db
10167 self.dquery = None
10168
10169
10170 if isinstance(query, dict):
10171 query = self.parse(query)
10172
10173 if not ignore_common_filters is None and \
10174 use_common_filters(query) == ignore_common_filters:
10175 query = copy.copy(query)
10176 query.ignore_common_filters = ignore_common_filters
10177 self.query = query
10178
10181
10182 - def __call__(self, query, ignore_common_filters=False):
10183 if query is None:
10184 return self
10185 elif isinstance(query,Table):
10186 query = self.db._adapter.id_query(query)
10187 elif isinstance(query,str):
10188 query = Expression(self.db,query)
10189 elif isinstance(query,Field):
10190 query = query!=None
10191 if self.query:
10192 return Set(self.db, self.query & query,
10193 ignore_common_filters=ignore_common_filters)
10194 else:
10195 return Set(self.db, query,
10196 ignore_common_filters=ignore_common_filters)
10197
10198 - def _count(self,distinct=None):
10199 return self.db._adapter._count(self.query,distinct)
10200
10201 - def _select(self, *fields, **attributes):
10202 adapter = self.db._adapter
10203 tablenames = adapter.tables(self.query,
10204 attributes.get('join',None),
10205 attributes.get('left',None),
10206 attributes.get('orderby',None),
10207 attributes.get('groupby',None))
10208 fields = adapter.expand_all(fields, tablenames)
10209 return adapter._select(self.query,fields,attributes)
10210
10212 db = self.db
10213 tablename = db._adapter.get_table(self.query)
10214 return db._adapter._delete(tablename,self.query)
10215
10216 - def _update(self, **update_fields):
10221
10222 - def as_dict(self, flat=False, sanitize=True):
10223 if flat:
10224 uid = dbname = uri = None
10225 codec = self.db._db_codec
10226 if not sanitize:
10227 uri, dbname, uid = (self.db._dbname, str(self.db),
10228 self.db._db_uid)
10229 d = {"query": self.query.as_dict(flat=flat)}
10230 d["db"] = {"uid": uid, "codec": codec,
10231 "name": dbname, "uri": uri}
10232 return d
10233 else: return self.__dict__
10234
10235 - def as_xml(self, sanitize=True):
10242
10243 - def as_json(self, sanitize=True):
10250
10251 - def parse(self, dquery):
10252 "Experimental: Turn a dictionary into a Query object"
10253 self.dquery = dquery
10254 return self.build(self.dquery)
10255
10257 "Experimental: see .parse()"
10258 op, first, second = (d["op"], d["first"],
10259 d.get("second", None))
10260 left = right = built = None
10261
10262 if op in ("AND", "OR"):
10263 if not (type(first), type(second)) == (dict, dict):
10264 raise SyntaxError("Invalid AND/OR query")
10265 if op == "AND":
10266 built = self.build(first) & self.build(second)
10267 else: built = self.build(first) | self.build(second)
10268
10269 elif op == "NOT":
10270 if first is None:
10271 raise SyntaxError("Invalid NOT query")
10272 built = ~self.build(first)
10273 else:
10274
10275 for k, v in {"left": first, "right": second}.items():
10276 if isinstance(v, dict) and v.get("op"):
10277 v = self.build(v)
10278 if isinstance(v, dict) and ("tablename" in v):
10279 v = self.db[v["tablename"]][v["fieldname"]]
10280 if k == "left": left = v
10281 else: right = v
10282
10283 if hasattr(self.db._adapter, op):
10284 opm = getattr(self.db._adapter, op)
10285
10286 if op == "EQ": built = left == right
10287 elif op == "NE": built = left != right
10288 elif op == "GT": built = left > right
10289 elif op == "GE": built = left >= right
10290 elif op == "LT": built = left < right
10291 elif op == "LE": built = left <= right
10292 elif op in ("JOIN", "LEFT_JOIN", "RANDOM", "ALLOW_NULL"):
10293 built = Expression(self.db, opm)
10294 elif op in ("LOWER", "UPPER", "EPOCH", "PRIMARY_KEY",
10295 "COALESCE_ZERO", "RAW", "INVERT"):
10296 built = Expression(self.db, opm, left)
10297 elif op in ("COUNT", "EXTRACT", "AGGREGATE", "SUBSTRING",
10298 "REGEXP", "LIKE", "ILIKE", "STARTSWITH",
10299 "ENDSWITH", "ADD", "SUB", "MUL", "DIV",
10300 "MOD", "AS", "ON", "COMMA", "NOT_NULL",
10301 "COALESCE", "CONTAINS", "BELONGS"):
10302 built = Expression(self.db, opm, left, right)
10303
10304 elif not (left or right): built = Expression(self.db, op)
10305 else:
10306 raise SyntaxError("Operator not supported: %s" % op)
10307
10308 return built
10309
10311 return not self.select(limitby=(0,1), orderby_on_limitby=False)
10312
10313 - def count(self,distinct=None, cache=None):
10314 db = self.db
10315 if cache:
10316 cache_model, time_expire = cache
10317 sql = self._count(distinct=distinct)
10318 key = db._uri + '/' + sql
10319 if len(key)>200: key = hashlib_md5(key).hexdigest()
10320 return cache_model(
10321 key,
10322 (lambda self=self,distinct=distinct: \
10323 db._adapter.count(self.query,distinct)),
10324 time_expire)
10325 return db._adapter.count(self.query,distinct)
10326
10327 - def select(self, *fields, **attributes):
10328 adapter = self.db._adapter
10329 tablenames = adapter.tables(self.query,
10330 attributes.get('join',None),
10331 attributes.get('left',None),
10332 attributes.get('orderby',None),
10333 attributes.get('groupby',None))
10334 fields = adapter.expand_all(fields, tablenames)
10335 return adapter.select(self.query,fields,attributes)
10336
10339
10341 db = self.db
10342 tablename = db._adapter.get_table(self.query)
10343 table = db[tablename]
10344 if any(f(self) for f in table._before_delete): return 0
10345 ret = db._adapter.delete(tablename,self.query)
10346 ret and [f(self) for f in table._after_delete]
10347 return ret
10348
10349 - def update(self, **update_fields):
10350 db = self.db
10351 tablename = db._adapter.get_table(self.query)
10352 table = db[tablename]
10353 table._attempt_upload(update_fields)
10354 if any(f(self,update_fields) for f in table._before_update):
10355 return 0
10356 fields = table._listify(update_fields,update=True)
10357 if not fields:
10358 raise SyntaxError("No fields to update")
10359 ret = db._adapter.update("%s" % table,self.query,fields)
10360 ret and [f(self,update_fields) for f in table._after_update]
10361 return ret
10362
10364 """
10365 same as update but does not call table._before_update and _after_update
10366 """
10367 tablename = self.db._adapter.get_table(self.query)
10368 table = self.db[tablename]
10369 fields = table._listify(update_fields,update=True)
10370 if not fields: raise SyntaxError("No fields to update")
10371
10372 ret = self.db._adapter.update("%s" % table,self.query,fields)
10373 return ret
10374
10376 tablename = self.db._adapter.get_table(self.query)
10377 response = Row()
10378 response.errors = Row()
10379 new_fields = copy.copy(update_fields)
10380 for key,value in update_fields.iteritems():
10381 value,error = self.db[tablename][key].validate(value)
10382 if error:
10383 response.errors[key] = error
10384 else:
10385 new_fields[key] = value
10386 table = self.db[tablename]
10387 if response.errors:
10388 response.updated = None
10389 else:
10390 if not any(f(self,new_fields) for f in table._before_update):
10391 fields = table._listify(new_fields,update=True)
10392 if not fields: raise SyntaxError("No fields to update")
10393 ret = self.db._adapter.update(tablename,self.query,fields)
10394 ret and [f(self,new_fields) for f in table._after_update]
10395 else:
10396 ret = 0
10397 response.updated = ret
10398 return response
10399
10401 table = self.db[self.db._adapter.tables(self.query)[0]]
10402
10403 if upload_fields:
10404 fields = upload_fields.keys()
10405 else:
10406 fields = table.fields
10407 fields = [f for f in fields if table[f].type == 'upload'
10408 and table[f].uploadfield == True
10409 and table[f].autodelete]
10410 if not fields:
10411 return False
10412 for record in self.select(*[table[f] for f in fields]):
10413 for fieldname in fields:
10414 field = table[fieldname]
10415 oldname = record.get(fieldname, None)
10416 if not oldname:
10417 continue
10418 if upload_fields and oldname == upload_fields[fieldname]:
10419 continue
10420 if field.custom_delete:
10421 field.custom_delete(oldname)
10422 else:
10423 uploadfolder = field.uploadfolder
10424 if not uploadfolder:
10425 uploadfolder = pjoin(
10426 self.db._adapter.folder, '..', 'uploads')
10427 if field.uploadseparate:
10428 items = oldname.split('.')
10429 uploadfolder = pjoin(
10430 uploadfolder,
10431 "%s.%s" % (items[0], items[1]),
10432 items[2][:2])
10433 oldpath = pjoin(uploadfolder, oldname)
10434 if exists(oldpath):
10435 os.unlink(oldpath)
10436 return False
10437
10439 - def __init__(self, colset, table, id):
10440 self.colset, self.db, self.tablename, self.id = \
10441 colset, table._db, table._tablename, id
10442
10444 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
10445 table = db[tablename]
10446 newfields = fields or dict(colset)
10447 for fieldname in newfields.keys():
10448 if not fieldname in table.fields or table[fieldname].type=='id':
10449 del newfields[fieldname]
10450 table._db(table._id==id,ignore_common_filters=True).update(**newfields)
10451 colset.update(newfields)
10452 return colset
10453
10456 self.db, self.tablename, self.id = table._db, table._tablename, id
10458 return self.db(self.db[self.tablename]._id==self.id).delete()
10459
10462 self.db, self.tablename, self.id = table._db, table._tablename, id
10464 if self.db._lazy_tables is False:
10465 raise AttributeError()
10466 table = self.db[self.tablename]
10467 other_table = self.db[other_tablename]
10468 for rfield in table._referenced_by:
10469 if rfield.table == other_table:
10470 return LazySet(rfield, self.id)
10471
10472 raise AttributeError()
10473
10476 self.db, self.tablename, self.fieldname, self.id = \
10477 field.db, field._tablename, field.name, id
10479 query = self.db[self.tablename][self.fieldname]==self.id
10480 return Set(self.db,query)
10483 - def __call__(self, query, ignore_common_filters=False):
10484 return self._getset()(query, ignore_common_filters)
10485 - def _count(self,distinct=None):
10487 - def _select(self, *fields, **attributes):
10491 - def _update(self, **update_fields):
10495 - def count(self,distinct=None, cache=None):
10497 - def select(self, *fields, **attributes):
10503 - def update(self, **update_fields):
10511
10514 self.method=method
10515 self.row=row
10517 return self.method(self.row,*args,**kwargs)
10518
10520 f.__lazy__ = True
10521 return f
10522
10523 -class Rows(object):
10524
10525 """
10526 A wrapper for the return value of a select. It basically represents a table.
10527 It has an iterator and each row is represented as a dictionary.
10528 """
10529
10530
10531
10532 - def __init__(
10533 self,
10534 db=None,
10535 records=[],
10536 colnames=[],
10537 compact=True,
10538 rawrows=None
10539 ):
10540 self.db = db
10541 self.records = records
10542 self.colnames = colnames
10543 self.compact = compact
10544 self.response = rawrows
10545
10547 return '<Rows (%s)>' % len(self.records)
10548
10550 """
10551 db.define_table('x',Field('number','integer'))
10552 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
10553
10554 from gluon.dal import lazy_virtualfield
10555
10556 class MyVirtualFields(object):
10557 # normal virtual field (backward compatible, discouraged)
10558 def normal_shift(self): return self.x.number+1
10559 # lazy virtual field (because of @staticmethod)
10560 @lazy_virtualfield
10561 def lazy_shift(instance,row,delta=4): return row.x.number+delta
10562 db.x.virtualfields.append(MyVirtualFields())
10563
10564 for row in db(db.x).select():
10565 print row.number, row.normal_shift, row.lazy_shift(delta=7)
10566 """
10567 if not keyed_virtualfields:
10568 return self
10569 for row in self.records:
10570 for (tablename,virtualfields) in keyed_virtualfields.iteritems():
10571 attributes = dir(virtualfields)
10572 if not tablename in row:
10573 box = row[tablename] = Row()
10574 else:
10575 box = row[tablename]
10576 updated = False
10577 for attribute in attributes:
10578 if attribute[0] != '_':
10579 method = getattr(virtualfields,attribute)
10580 if hasattr(method,'__lazy__'):
10581 box[attribute]=VirtualCommand(method,row)
10582 elif type(method)==types.MethodType:
10583 if not updated:
10584 virtualfields.__dict__.update(row)
10585 updated = True
10586 box[attribute]=method()
10587 return self
10588
10590 if self.colnames!=other.colnames:
10591 raise Exception('Cannot & incompatible Rows objects')
10592 records = self.records+other.records
10593 return Rows(self.db,records,self.colnames)
10594
10596 if self.colnames!=other.colnames:
10597 raise Exception('Cannot | incompatible Rows objects')
10598 records = [record for record in other.records
10599 if not record in self.records]
10600 records = self.records + records
10601 return Rows(self.db,records,self.colnames)
10602
10604 if len(self.records):
10605 return 1
10606 return 0
10607
10609 return len(self.records)
10610
10612 return Rows(self.db,self.records[a:b],self.colnames,compact=self.compact)
10613
10615 row = self.records[i]
10616 keys = row.keys()
10617 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10618 return row[row.keys()[0]]
10619 return row
10620
10622 """
10623 iterator over records
10624 """
10625
10626 for i in xrange(len(self)):
10627 yield self[i]
10628
10630 """
10631 serializes the table into a csv file
10632 """
10633
10634 s = StringIO.StringIO()
10635 self.export_to_csv_file(s)
10636 return s.getvalue()
10637
10639 if not self.records:
10640 return None
10641 return self[0]
10642
10644 if not self.records:
10645 return None
10646 return self[-1]
10647
10648 - def find(self,f,limitby=None):
10649 """
10650 returns a new Rows object, a subset of the original object,
10651 filtered by the function f
10652 """
10653 if not self:
10654 return Rows(self.db, [], self.colnames)
10655 records = []
10656 if limitby:
10657 a,b = limitby
10658 else:
10659 a,b = 0,len(self)
10660 k = 0
10661 for row in self:
10662 if f(row):
10663 if a<=k: records.append(row)
10664 k += 1
10665 if k==b: break
10666 return Rows(self.db, records, self.colnames)
10667
10669 """
10670 removes elements from the calling Rows object, filtered by the function f,
10671 and returns a new Rows object containing the removed elements
10672 """
10673 if not self.records:
10674 return Rows(self.db, [], self.colnames)
10675 removed = []
10676 i=0
10677 while i<len(self):
10678 row = self[i]
10679 if f(row):
10680 removed.append(self.records[i])
10681 del self.records[i]
10682 else:
10683 i += 1
10684 return Rows(self.db, removed, self.colnames)
10685
10686 - def sort(self, f, reverse=False):
10687 """
10688 returns a list of sorted elements (not sorted in place)
10689 """
10690 rows = Rows(self.db,[],self.colnames,compact=False)
10691 rows.records = sorted(self,key=f,reverse=reverse)
10692 return rows
10693
10695 """
10696 regroups the rows, by one of the fields
10697 """
10698 one_result = False
10699 if 'one_result' in args:
10700 one_result = args['one_result']
10701
10702 def build_fields_struct(row, fields, num, groups):
10703 ''' helper function:
10704 '''
10705 if num > len(fields)-1:
10706 if one_result:
10707 return row
10708 else:
10709 return [row]
10710
10711 key = fields[num]
10712 value = row[key]
10713
10714 if value not in groups:
10715 groups[value] = build_fields_struct(row, fields, num+1, {})
10716 else:
10717 struct = build_fields_struct(row, fields, num+1, groups[ value ])
10718
10719
10720 if type(struct) == type(dict()):
10721 groups[value].update()
10722
10723 elif type(struct) == type(list()):
10724 groups[value] += struct
10725
10726 else:
10727 groups[value] = struct
10728
10729 return groups
10730
10731 if len(fields) == 0:
10732 return self
10733
10734
10735 if not self.records:
10736 return {}
10737
10738 grouped_row_group = dict()
10739
10740
10741 for row in self:
10742 build_fields_struct(row, fields, 0, grouped_row_group)
10743
10744 return grouped_row_group
10745
10746 - def render(self, i=None, fields=None):
10747 """
10748 Takes an index and returns a copy of the indexed row with values
10749 transformed via the "represent" attributes of the associated fields.
10750
10751 If no index is specified, a generator is returned for iteration
10752 over all the rows.
10753
10754 fields -- a list of fields to transform (if None, all fields with
10755 "represent" attributes will be transformed).
10756 """
10757
10758
10759 if i is None:
10760 return (self.render(i, fields=fields) for i in range(len(self)))
10761 import sqlhtml
10762 row = copy.deepcopy(self.records[i])
10763 keys = row.keys()
10764 tables = [f.tablename for f in fields] if fields \
10765 else [k for k in keys if k != '_extra']
10766 for table in tables:
10767 repr_fields = [f.name for f in fields if f.tablename == table] \
10768 if fields else [k for k in row[table].keys()
10769 if (hasattr(self.db[table], k) and
10770 isinstance(self.db[table][k], Field)
10771 and self.db[table][k].represent)]
10772 for field in repr_fields:
10773 row[table][field] = sqlhtml.represent(
10774 self.db[table][field], row[table][field], row[table])
10775 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10776 return row[keys[0]]
10777 return row
10778
10779 - def as_list(self,
10780 compact=True,
10781 storage_to_dict=True,
10782 datetime_to_str=False,
10783 custom_types=None):
10784 """
10785 returns the data as a list or dictionary.
10786 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10787 :param datetime_to_str: convert datetime fields as strings (default False)
10788 """
10789 (oc, self.compact) = (self.compact, compact)
10790 if storage_to_dict:
10791 items = [item.as_dict(datetime_to_str, custom_types) for item in self]
10792 else:
10793 items = [item for item in self]
10794 self.compact = compact
10795 return items
10796
10797
10798 - def as_dict(self,
10799 key='id',
10800 compact=True,
10801 storage_to_dict=True,
10802 datetime_to_str=False,
10803 custom_types=None):
10804 """
10805 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
10806
10807 :param key: the name of the field to be used as dict key, normally the id
10808 :param compact: ? (default True)
10809 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10810 :param datetime_to_str: convert datetime fields as strings (default False)
10811 """
10812
10813
10814 multi = False
10815 f = self.first()
10816 if f and isinstance(key, basestring):
10817 multi = any([isinstance(v, f.__class__) for v in f.values()])
10818 if (not "." in key) and multi:
10819
10820 def new_key():
10821 i = 0
10822 while True:
10823 yield i
10824 i += 1
10825 key_generator = new_key()
10826 key = lambda r: key_generator.next()
10827
10828 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types)
10829 if isinstance(key,str) and key.count('.')==1:
10830 (table, field) = key.split('.')
10831 return dict([(r[table][field],r) for r in rows])
10832 elif isinstance(key,str):
10833 return dict([(r[key],r) for r in rows])
10834 else:
10835 return dict([(key(r),r) for r in rows])
10836
10837
10838 - def as_trees(self, parent_name='parent_id', children_name='children'):
10839 roots = []
10840 drows = {}
10841 for row in self:
10842 drows[row.id] = row
10843 row[children_name] = []
10844 for row in self:
10845 parent = row[parent_name]
10846 if parent is None:
10847 roots.append(row)
10848 else:
10849 drows[parent][children_name].append(row)
10850 return roots
10851
10853 """
10854 export data to csv, the first line contains the column names
10855
10856 :param ofile: where the csv must be exported to
10857 :param null: how null values must be represented (default '<NULL>')
10858 :param delimiter: delimiter to separate values (default ',')
10859 :param quotechar: character to use to quote string values (default '"')
10860 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
10861 :param represent: use the fields .represent value (default False)
10862 :param colnames: list of column names to use (default self.colnames)
10863 This will only work when exporting rows objects!!!!
10864 DO NOT use this with db.export_to_csv()
10865 """
10866 delimiter = kwargs.get('delimiter', ',')
10867 quotechar = kwargs.get('quotechar', '"')
10868 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
10869 represent = kwargs.get('represent', False)
10870 writer = csv.writer(ofile, delimiter=delimiter,
10871 quotechar=quotechar, quoting=quoting)
10872 colnames = kwargs.get('colnames', self.colnames)
10873 write_colnames = kwargs.get('write_colnames',True)
10874
10875 if write_colnames:
10876 writer.writerow(colnames)
10877
10878 def none_exception(value):
10879 """
10880 returns a cleaned up value that can be used for csv export:
10881 - unicode text is encoded as such
10882 - None values are replaced with the given representation (default <NULL>)
10883 """
10884 if value is None:
10885 return null
10886 elif isinstance(value, unicode):
10887 return value.encode('utf8')
10888 elif isinstance(value,Reference):
10889 return long(value)
10890 elif hasattr(value, 'isoformat'):
10891 return value.isoformat()[:19].replace('T', ' ')
10892 elif isinstance(value, (list,tuple)):
10893 return bar_encode(value)
10894 return value
10895
10896 for record in self:
10897 row = []
10898 for col in colnames:
10899 if not REGEX_TABLE_DOT_FIELD.match(col):
10900 row.append(record._extra[col])
10901 else:
10902 (t, f) = col.split('.')
10903 field = self.db[t][f]
10904 if isinstance(record.get(t, None), (Row,dict)):
10905 value = record[t][f]
10906 else:
10907 value = record[f]
10908 if field.type=='blob' and not value is None:
10909 value = base64.b64encode(value)
10910 elif represent and field.represent:
10911 value = field.represent(value)
10912 row.append(none_exception(value))
10913 writer.writerow(row)
10914
10915 - def xml(self,strict=False,row_name='row',rows_name='rows'):
10916 """
10917 serializes the table using sqlhtml.SQLTABLE (if present)
10918 """
10919
10920 if strict:
10921 ncols = len(self.colnames)
10922 return '<%s>\n%s\n</%s>' % (rows_name,
10923 '\n'.join(row.as_xml(row_name=row_name,
10924 colnames=self.colnames) for
10925 row in self), rows_name)
10926
10927 import sqlhtml
10928 return sqlhtml.SQLTABLE(self).xml()
10929
10930 - def as_xml(self,row_name='row',rows_name='rows'):
10931 return self.xml(strict=True, row_name=row_name, rows_name=rows_name)
10932
10933 - def as_json(self, mode='object', default=None):
10934 """
10935 serializes the rows to a JSON list or object with objects
10936 mode='object' is not implemented (should return a nested
10937 object structure)
10938 """
10939
10940 items = [record.as_json(mode=mode, default=default,
10941 serialize=False,
10942 colnames=self.colnames) for
10943 record in self]
10944
10945 if have_serializers:
10946 return serializers.json(items,
10947 default=default or
10948 serializers.custom_json)
10949 elif simplejson:
10950 return simplejson.dumps(items)
10951 else:
10952 raise RuntimeError("missing simplejson")
10953
10954
10955 as_csv = __str__
10956 json = as_json
10957
10958
10959
10960
10961
10962
10963 -def test_all():
10964 """
10965
10966 >>> if len(sys.argv)<2: db = DAL("sqlite://test.db")
10967 >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
10968 >>> tmp = db.define_table('users',\
10969 Field('stringf', 'string', length=32, required=True),\
10970 Field('booleanf', 'boolean', default=False),\
10971 Field('passwordf', 'password', notnull=True),\
10972 Field('uploadf', 'upload'),\
10973 Field('blobf', 'blob'),\
10974 Field('integerf', 'integer', unique=True),\
10975 Field('doublef', 'double', unique=True,notnull=True),\
10976 Field('jsonf', 'json'),\
10977 Field('datef', 'date', default=datetime.date.today()),\
10978 Field('timef', 'time'),\
10979 Field('datetimef', 'datetime'),\
10980 migrate='test_user.table')
10981
10982 Insert a field
10983
10984 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
10985 uploadf=None, integerf=5, doublef=3.14,\
10986 jsonf={"j": True},\
10987 datef=datetime.date(2001, 1, 1),\
10988 timef=datetime.time(12, 30, 15),\
10989 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
10990 1
10991
10992 Drop the table
10993
10994 >>> db.users.drop()
10995
10996 Examples of insert, select, update, delete
10997
10998 >>> tmp = db.define_table('person',\
10999 Field('name'),\
11000 Field('birth','date'),\
11001 migrate='test_person.table')
11002 >>> person_id = db.person.insert(name='Marco',birth='2005-06-22')
11003 >>> person_id = db.person.insert(name='Massimo',birth='1971-12-21')
11004
11005 commented len(db().select(db.person.ALL))
11006 commented 2
11007
11008 >>> me = db(db.person.id==person_id).select()[0] # test select
11009 >>> me.name
11010 'Massimo'
11011 >>> db.person[2].name
11012 'Massimo'
11013 >>> db.person(2).name
11014 'Massimo'
11015 >>> db.person(name='Massimo').name
11016 'Massimo'
11017 >>> db.person(db.person.name=='Massimo').name
11018 'Massimo'
11019 >>> row = db.person[2]
11020 >>> row.name == row['name'] == row['person.name'] == row('person.name')
11021 True
11022 >>> db(db.person.name=='Massimo').update(name='massimo') # test update
11023 1
11024 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
11025 1
11026
11027 Update a single record
11028
11029 >>> me.update_record(name="Max")
11030 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}>
11031 >>> me.name
11032 'Max'
11033
11034 Examples of complex search conditions
11035
11036 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
11037 1
11038 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
11039 1
11040 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
11041 1
11042 >>> me = db(db.person.id==person_id).select(db.person.name)[0]
11043 >>> me.name
11044 'Max'
11045
11046 Examples of search conditions using extract from date/datetime/time
11047
11048 >>> len(db(db.person.birth.month()==12).select())
11049 1
11050 >>> len(db(db.person.birth.year()>1900).select())
11051 1
11052
11053 Example of usage of NULL
11054
11055 >>> len(db(db.person.birth==None).select()) ### test NULL
11056 0
11057 >>> len(db(db.person.birth!=None).select()) ### test NULL
11058 1
11059
11060 Examples of search conditions using lower, upper, and like
11061
11062 >>> len(db(db.person.name.upper()=='MAX').select())
11063 1
11064 >>> len(db(db.person.name.like('%ax')).select())
11065 1
11066 >>> len(db(db.person.name.upper().like('%AX')).select())
11067 1
11068 >>> len(db(~db.person.name.upper().like('%AX')).select())
11069 0
11070
11071 orderby, groupby and limitby
11072
11073 >>> people = db().select(db.person.name, orderby=db.person.name)
11074 >>> order = db.person.name|~db.person.birth
11075 >>> people = db().select(db.person.name, orderby=order)
11076
11077 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
11078
11079 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
11080
11081 Example of one 2 many relation
11082
11083 >>> tmp = db.define_table('dog',\
11084 Field('name'),\
11085 Field('birth','date'),\
11086 Field('owner',db.person),\
11087 migrate='test_dog.table')
11088 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
11089 1
11090
11091 A simple JOIN
11092
11093 >>> len(db(db.dog.owner==db.person.id).select())
11094 1
11095
11096 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
11097 1
11098
11099 Drop tables
11100
11101 >>> db.dog.drop()
11102 >>> db.person.drop()
11103
11104 Example of many 2 many relation and Set
11105
11106 >>> tmp = db.define_table('author', Field('name'),\
11107 migrate='test_author.table')
11108 >>> tmp = db.define_table('paper', Field('title'),\
11109 migrate='test_paper.table')
11110 >>> tmp = db.define_table('authorship',\
11111 Field('author_id', db.author),\
11112 Field('paper_id', db.paper),\
11113 migrate='test_authorship.table')
11114 >>> aid = db.author.insert(name='Massimo')
11115 >>> pid = db.paper.insert(title='QCD')
11116 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
11117
11118 Define a Set
11119
11120 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
11121 >>> rows = authored_papers.select(db.author.name, db.paper.title)
11122 >>> for row in rows: print row.author.name, row.paper.title
11123 Massimo QCD
11124
11125 Example of search condition using belongs
11126
11127 >>> set = (1, 2, 3)
11128 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
11129 >>> print rows[0].title
11130 QCD
11131
11132 Example of search condition using nested select
11133
11134 >>> nested_select = db()._select(db.authorship.paper_id)
11135 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
11136 >>> print rows[0].title
11137 QCD
11138
11139 Example of expressions
11140
11141 >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
11142 >>> db(mynumber).delete()
11143 0
11144 >>> for i in range(10): tmp = mynumber.insert(x=i)
11145 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum())
11146 45
11147
11148 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
11149 5
11150
11151 Output in csv
11152
11153 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
11154 author.name,paper.title\r
11155 Massimo,QCD
11156
11157 Delete all leftover tables
11158
11159 >>> DAL.distributed_transaction_commit(db)
11160
11161 >>> db.mynumber.drop()
11162 >>> db.authorship.drop()
11163 >>> db.author.drop()
11164 >>> db.paper.drop()
11165 """
11166
11167
11168
11169
11170 SQLField = Field
11171 SQLTable = Table
11172 SQLXorable = Expression
11173 SQLQuery = Query
11174 SQLSet = Set
11175 SQLRows = Rows
11176 SQLStorage = Row
11177 SQLDB = DAL
11178 GQLDB = DAL
11179 DAL.Field = Field
11180 DAL.Table = Table
11181
11182
11183
11184
11185
11186 -def geoPoint(x,y):
11187 return "POINT (%f %f)" % (x,y)
11188
11190 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
11191
11193 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
11194
11195
11196
11197
11198
11199 if __name__ == '__main__':
11200 import doctest
11201 doctest.testmod()
11202